Modify IBD to accept various transport backends

This allows IBD to try using faster and cheaper transports first with
fallback to more reliable and expensive transports later.

1 files changed, 34 insertions, 91 deletions

diff --git a/backend/internal/ibd/client.go b/backend/internal/ibd/client.go
index c1cbb8a..2b91268 100644
--- a/backend/internal/ibd/client.go
+++ b/backend/internal/ibd/client.go
@@ -2,56 +2,28 @@ package ibd
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
-	"fmt"
-	"io"
+	"log/slog"
 	"net/http"
-	"net/url"
-	"strconv"
+	"slices"
 
 	"github.com/ansg191/ibd-trader-backend/internal/database"
+	"github.com/ansg191/ibd-trader-backend/internal/ibd/transport"
 )
 
 var ErrNoAvailableCookies = errors.New("no available cookies")
+var ErrNoAvailableTransports = errors.New("no available transports")
 
 type Client struct {
-	// HTTP client used to make requests
-	client *http.Client
-	// Scrapfly API key
-	apiKey string
-	// Client-wide Scrape options
-	options ScrapeOptions
-	// Cookie source
-	cookies database.CookieSource
-	// Proxy URL for non-scrapfly requests
-	proxyUrl *url.URL
+	transports []transport.Transport
+	cookies    database.CookieSource
 }
 
 func NewClient(
-	client *http.Client,
-	apiKey string,
+	transports []transport.Transport,
 	cookies database.CookieSource,
-	proxyUrl string,
-	opts ...ScrapeOption,
-) (*Client, error) {
-	options := defaultScrapeOptions
-	for _, opt := range opts {
-		opt(&options)
-	}
-
-	pProxyUrl, err := url.Parse(proxyUrl)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Client{
-		client:   client,
-		options:  options,
-		apiKey:   apiKey,
-		cookies:  cookies,
-		proxyUrl: pProxyUrl,
-	}, nil
+) *Client {
+	return &Client{transports, cookies}
 }
 
 func (c *Client) getCookie(ctx context.Context, subject *string) (uint, *http.Cookie, error) {
@@ -83,64 +55,35 @@ func (c *Client) getCookie(ctx context.Context, subject *string) (uint, *http.Co
 	return cookie.ID, cookie.ToHTTPCookie(), nil
 }
 
-func (c *Client) Do(req *http.Request, opts ...ScrapeOption) (*ScraperResponse, error) {
-	options := c.options
-	for _, opt := range opts {
-		opt(&options)
-	}
-
-	// Construct scrape request URL
-	scrapeUrl, err := url.Parse(options.baseURL)
-	if err != nil {
-		panic(err)
-	}
-	scrapeUrl.RawQuery = c.constructRawQuery(options, req.URL, req.Header)
-
-	// Construct scrape request
-	scrapeReq, err := http.NewRequestWithContext(req.Context(), req.Method, scrapeUrl.String(), req.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	// Send scrape request
-	resp, err := c.client.Do(scrapeReq)
-	if err != nil {
-		return nil, err
-	}
-	defer func(Body io.ReadCloser) {
-		_ = Body.Close()
-	}(resp.Body)
-
-	// Parse scrape response
-	scraperResponse := new(ScraperResponse)
-	err = json.NewDecoder(resp.Body).Decode(scraperResponse)
-	if err != nil {
-		return nil, err
-	}
-
-	return scraperResponse, nil
+func (c *Client) Do(req *http.Request) (*http.Response, error) {
+	return c.DoWithStatus(req, []int{http.StatusOK})
 }
 
-func (c *Client) constructRawQuery(options ScrapeOptions, u *url.URL, headers http.Header) string {
-	params := url.Values{}
-	params.Set("key", c.apiKey)
-	params.Set("url", u.String())
-	if options.country != nil {
-		params.Set("country", *options.country)
-	}
-	params.Set("asp", strconv.FormatBool(options.asp))
-	params.Set("proxy_pool", options.proxyPool.String())
-	params.Set("render_js", strconv.FormatBool(options.renderJS))
-	params.Set("cache", strconv.FormatBool(options.cache))
-
-	for k, v := range headers {
-		for i, vv := range v {
-			params.Add(
-				fmt.Sprintf("headers[%s][%d]", k, i),
-				vv,
+func (c *Client) DoWithStatus(req *http.Request, expectedStatus []int) (*http.Response, error) {
+	for i, tp := range c.transports {
+		resp, err := tp.Do(req)
+		if errors.Is(err, transport.ErrUnsupportedRequest) {
+			// Skip unsupported transport
+			continue
+		}
+		if err != nil {
+			slog.ErrorContext(req.Context(), "transport error",
+				"transport", i,
+				"error", err,
+			)
+			continue
+		}
+		if slices.Contains(expectedStatus, resp.StatusCode) {
+			return resp, nil
+		} else {
+			slog.ErrorContext(req.Context(), "unexpected status code",
+				"transport", i,
+				"expected", expectedStatus,
+				"actual", resp.StatusCode,
 			)
+			continue
 		}
 	}
 
-	return params.Encode()
+	return nil, ErrNoAvailableTransports
 }