Modify IBD to accept various transport backends

This allows IBD to try using faster and cheaper transports first with
fallback to more reliable and expensive transports later.

1 files changed, 12 insertions, 142 deletions

diff --git a/backend/internal/ibd/client_test.go b/backend/internal/ibd/client_test.go
index 048ad59..0a8fa98 100644
--- a/backend/internal/ibd/client_test.go
+++ b/backend/internal/ibd/client_test.go
@@ -1,177 +1,50 @@
 package ibd
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 
 	"github.com/ansg191/ibd-trader-backend/internal/database"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-const apiKey = "test-api-key-123"
-
-func newServer(t *testing.T, handler http.Handler) *httptest.Server {
-	t.Helper()
-
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		r.Close = true
-		defer r.Body.Close()
-		req := reconstructReq(t, r)
-
-		rw := newResponseWriter()
-		handler.ServeHTTP(rw, req)
-		require.NoError(t, rw.Done(w))
-	}))
-}
-
-func reconstructReq(t *testing.T, r *http.Request) *http.Request {
-	t.Helper()
-
-	params := r.URL.Query()
-	require.Equal(t, apiKey, params.Get("key"))
-
-	// Reconstruct the request from the query params
-	var key string
-	var url string
-	headers := make(http.Header)
-	for k, v := range params {
-		switch k {
-		case "key":
-			key = v[0]
-		case "url":
-			url = v[0]
-		default:
-			if strings.HasPrefix(k, "headers") {
-				var name string
-				// Get index of first [
-				i := strings.Index(k, "[")
-				if i == -1 {
-					t.Fatalf("invalid header key: %s", k)
-				}
-				// Get index of first ]
-				j := strings.Index(k, "]")
-				if j == -1 {
-					t.Fatalf("invalid header key: %s", k)
-				}
-
-				// Get the header name
-				name = k[i+1 : j]
-				headers.Set(name, v[0])
-			}
-		}
-	}
-	require.Equal(t, apiKey, key)
-	require.NotEmpty(t, url)
-
-	req, err := http.NewRequestWithContext(r.Context(), r.Method, url, r.Body)
-	require.NoError(t, err)
-	req.Header = headers
-
-	return req
-}
-
-type responsewriter struct {
-	ret     ScraperResponse
-	body    bytes.Buffer
-	headers http.Header
-}
-
-func newResponseWriter() *responsewriter {
-	return &responsewriter{
-		headers: make(http.Header),
-	}
-}
-
-func (w *responsewriter) Header() http.Header {
-	return w.headers
-}
-
-func (w *responsewriter) Write(bytes []byte) (int, error) {
-	if w.ret.Result.StatusCode == 0 {
-		w.ret.Result.StatusCode = http.StatusOK
-	}
-	return w.body.Write(bytes)
-}
-
-func (w *responsewriter) WriteHeader(statusCode int) {
-	w.ret.Result.StatusCode = statusCode
-}
-
-func (w *responsewriter) Done(rw http.ResponseWriter) error {
-	w.ret.Result.Content = w.body.String()
-
-	w.ret.Result.ResponseHeaders = make(map[string]string)
-	for k, v := range w.headers {
-		if k == "Set-Cookie" {
-			continue
-		}
-		w.ret.Result.ResponseHeaders[k] = v[0]
-	}
-
-	req := http.Response{Header: w.headers}
-	w.ret.Result.Cookies = make([]ScraperCookie, 0)
-	for _, c := range req.Cookies() {
-		var cookie ScraperCookie
-		cookie.FromHTTPCookie(c)
-		w.ret.Result.Cookies = append(w.ret.Result.Cookies, cookie)
-	}
-
-	rw.WriteHeader(http.StatusOK)
-	return json.NewEncoder(rw).Encode(w.ret)
-}
-
 func TestClient_getCookie(t *testing.T) {
 	t.Parallel()
 
 	t.Run("no cookies", func(t *testing.T) {
 		t.Parallel()
 
-		client, err := NewClient(
-			http.DefaultClient,
-			apiKey,
+		client := NewClient(
+			nil,
 			new(emptyCookieSourceStub),
-			"",
 		)
-		require.NoError(t, err)
 
-		_, _, err = client.getCookie(context.Background(), nil)
+		_, _, err := client.getCookie(context.Background(), nil)
 		assert.ErrorIs(t, err, ErrNoAvailableCookies)
 	})
 
 	t.Run("no cookies by subject", func(t *testing.T) {
 		t.Parallel()
 
-		client, err := NewClient(
-			http.DefaultClient,
-			apiKey,
+		client := NewClient(
+			nil,
 			new(emptyCookieSourceStub),
-			"",
 		)
-		require.NoError(t, err)
 
 		subject := "test"
-		_, _, err = client.getCookie(context.Background(), &subject)
+		_, _, err := client.getCookie(context.Background(), &subject)
 		assert.ErrorIs(t, err, ErrNoAvailableCookies)
 	})
 
 	t.Run("get any cookie", func(t *testing.T) {
 		t.Parallel()
 
-		client, err := NewClient(
-			http.DefaultClient,
-			apiKey,
-			new(cookieSourceStub),
-			"",
+		client := NewClient(
+			nil,
+			new(emptyCookieSourceStub),
 		)
-		require.NoError(t, err)
 
 		id, cookie, err := client.getCookie(context.Background(), nil)
 		require.NoError(t, err)
@@ -186,13 +59,10 @@ func TestClient_getCookie(t *testing.T) {
 	t.Run("get cookie by subject", func(t *testing.T) {
 		t.Parallel()
 
-		client, err := NewClient(
-			http.DefaultClient,
-			apiKey,
-			new(cookieSourceStub),
-			"",
+		client := NewClient(
+			nil,
+			new(emptyCookieSourceStub),
 		)
-		require.NoError(t, err)
 
 		subject := "test"
 		id, cookie, err := client.getCookie(context.Background(), &subject)