diff options
Diffstat (limited to 'backend/internal/ibd/options.go')
-rw-r--r-- | backend/internal/ibd/options.go | 84 |
1 files changed, 84 insertions, 0 deletions
diff --git a/backend/internal/ibd/options.go b/backend/internal/ibd/options.go new file mode 100644 index 0000000..a07241e --- /dev/null +++ b/backend/internal/ibd/options.go @@ -0,0 +1,84 @@ +package ibd + +const BaseURL = "https://api.scrapfly.io/scrape" + +var defaultScrapeOptions = ScrapeOptions{ + baseURL: BaseURL, + country: nil, + asp: true, + proxyPool: ProxyPoolDatacenter, + renderJS: false, + cache: false, +} + +type ScrapeOption func(*ScrapeOptions) + +type ScrapeOptions struct { + baseURL string + country *string + asp bool + proxyPool ProxyPool + renderJS bool + cache bool + debug bool +} + +type ProxyPool uint8 + +const ( + ProxyPoolDatacenter ProxyPool = iota + ProxyPoolResidential +) + +func (p ProxyPool) String() string { + switch p { + case ProxyPoolDatacenter: + return "public_datacenter_pool" + case ProxyPoolResidential: + return "public_residential_pool" + default: + panic("invalid proxy pool") + } +} + +func WithCountry(country string) ScrapeOption { + return func(o *ScrapeOptions) { + o.country = &country + } +} + +func WithASP(asp bool) ScrapeOption { + return func(o *ScrapeOptions) { + o.asp = asp + } +} + +func WithProxyPool(proxyPool ProxyPool) ScrapeOption { + return func(o *ScrapeOptions) { + o.proxyPool = proxyPool + } +} + +func WithRenderJS(jsRender bool) ScrapeOption { + return func(o *ScrapeOptions) { + o.renderJS = jsRender + } +} + +func WithCache(cache bool) ScrapeOption { + return func(o *ScrapeOptions) { + o.cache = cache + } +} + +func WithDebug(debug bool) ScrapeOption { + return func(o *ScrapeOptions) { + o.debug = debug + } +} + +func WithBaseURL(baseURL string) ScrapeOption { + return func(o *ScrapeOptions) { + o.baseURL = baseURL + } +} |