package ibd import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strconv" "github.com/ansg191/ibd-trader-backend/internal/database" ) var ErrNoAvailableCookies = errors.New("no available cookies") type Client struct { // HTTP client used to make requests client *http.Client // Scrapfly API key apiKey string // Client-wide Scrape options options ScrapeOptions // Cookie source cookies database.CookieSource // Proxy URL for non-scrapfly requests proxyUrl *url.URL } func NewClient( client *http.Client, apiKey string, cookies database.CookieSource, proxyUrl string, opts ...ScrapeOption, ) (*Client, error) { options := defaultScrapeOptions for _, opt := range opts { opt(&options) } pProxyUrl, err := url.Parse(proxyUrl) if err != nil { return nil, err } return &Client{ client: client, options: options, apiKey: apiKey, cookies: cookies, proxyUrl: pProxyUrl, }, nil } func (c *Client) getCookie(ctx context.Context, subject *string) (uint, *http.Cookie, error) { if subject == nil { // No subject requirement, get any cookie cookie, err := c.cookies.GetAnyCookie(ctx) if err != nil { return 0, nil, err } if cookie == nil { return 0, nil, ErrNoAvailableCookies } return cookie.ID, cookie.ToHTTPCookie(), nil } // Get cookie by subject cookies, err := c.cookies.GetCookies(ctx, *subject, false) if err != nil { return 0, nil, err } if len(cookies) == 0 { return 0, nil, ErrNoAvailableCookies } cookie := cookies[0] return cookie.ID, cookie.ToHTTPCookie(), nil } func (c *Client) Do(req *http.Request, opts ...ScrapeOption) (*ScraperResponse, error) { options := c.options for _, opt := range opts { opt(&options) } // Construct scrape request URL scrapeUrl, err := url.Parse(options.baseURL) if err != nil { panic(err) } scrapeUrl.RawQuery = c.constructRawQuery(options, req.URL, req.Header) // Construct scrape request scrapeReq, err := http.NewRequestWithContext(req.Context(), req.Method, scrapeUrl.String(), req.Body) if err != nil { return nil, err } // Send scrape request resp, err := c.client.Do(scrapeReq) if err != nil { return nil, err } defer func(Body io.ReadCloser) { _ = Body.Close() }(resp.Body) // Parse scrape response scraperResponse := new(ScraperResponse) err = json.NewDecoder(resp.Body).Decode(scraperResponse) if err != nil { return nil, err } return scraperResponse, nil } func (c *Client) constructRawQuery(options ScrapeOptions, u *url.URL, headers http.Header) string { params := url.Values{} params.Set("key", c.apiKey) params.Set("url", u.String()) if options.country != nil { params.Set("country", *options.country) } params.Set("asp", strconv.FormatBool(options.asp)) params.Set("proxy_pool", options.proxyPool.String()) params.Set("render_js", strconv.FormatBool(options.renderJS)) params.Set("cache", strconv.FormatBool(options.cache)) for k, v := range headers { for i, vv := range v { params.Add( fmt.Sprintf("headers[%s][%d]", k, i), vv, ) } } return params.Encode() }