Improve selection of IBD transports

16 files changed, 249 insertions, 50 deletions

diff --git a/backend/api b/backend/api
-Subproject 75ae31c95e6cbba285785e6029c9de93ff6a47b
+Subproject 250159ea00457090d9cc7b3cc270618d14ce9a9
diff --git a/backend/cmd/main.go b/backend/cmd/main.go
index 16fc1ce..6b49980 100644
--- a/backend/cmd/main.go
+++ b/backend/cmd/main.go
@@ -154,10 +154,10 @@ func setupIBDClient(cfg *config.Config, db database.Database) (*ibd.Client, erro
 	t := http.DefaultTransport.(*http.Transport).Clone()
 	t.Proxy = http.ProxyURL(pUrl)
 	transports := []transport.Transport{
-		&http.Client{Transport: t},                       // Default proxied transport
-		scrapfly.New(http.DefaultClient, cfg.IBD.APIKey), // Scrapfly transport
+		transport.NewStandardTransport(&http.Client{Transport: t}), // Default proxied transport
+		scrapfly.New(http.DefaultClient, cfg.IBD.APIKey),           // Scrapfly transport
 	}
-	client := ibd.NewClient(transports, db)
+	client := ibd.NewClient(db, transports...)
 	return client, nil
 }
 
diff --git a/backend/go.mod b/backend/go.mod
index bd55e54..cdd1e1c 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -7,6 +7,7 @@ toolchain go1.22.5
 require (
 	cloud.google.com/go/kms v1.18.4
 	cloud.google.com/go/longrunning v0.5.11
+	github.com/EDDYCJY/fake-useragent v0.2.0
 	github.com/Rhymond/go-money v1.0.14
 	github.com/bsm/redislock v0.9.4
 	github.com/buraksezer/consistent v0.10.0
@@ -37,6 +38,8 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
 	cloud.google.com/go/compute/metadata v0.5.0 // indirect
 	cloud.google.com/go/iam v1.1.12 // indirect
+	github.com/PuerkitoBio/goquery v1.9.2 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index 3f5e220..d7235e3 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -395,11 +395,17 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/EDDYCJY/fake-useragent v0.2.0 h1:Jcnkk2bgXmDpX0z+ELlUErTkoLb/mxFBNd2YdcpvJBs=
+github.com/EDDYCJY/fake-useragent v0.2.0/go.mod h1:5wn3zzlDxhKW6NYknushqinPcAqZcAPHy8lLczCdJdc=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
+github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
 github.com/Rhymond/go-money v1.0.14 h1:HtdIZ0mP4LrnpN3wdRhsik7pool7x22ILZdDe3moL6E=
 github.com/Rhymond/go-money v1.0.14/go.mod h1:iHvCuIvitxu2JIlAlhF0g9jHqjRSr+rpdOs7Omqlupg=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
@@ -746,6 +752,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
 golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -795,6 +802,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
 golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -903,10 +912,14 @@ golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
 golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -918,6 +931,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
 golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -979,6 +994,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
 golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/backend/internal/ibd/auth.go b/backend/internal/ibd/auth.go
index f09f3f7..7b82057 100644
--- a/backend/internal/ibd/auth.go
+++ b/backend/internal/ibd/auth.go
@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/ansg191/ibd-trader-backend/internal/ibd/transport"
 	"golang.org/x/net/html"
 )
 
@@ -48,7 +49,7 @@ func (c *Client) getLoginPage(ctx context.Context) (*authConfig, error) {
 		return nil, err
 	}
 
-	resp, err := c.Do(req)
+	resp, err := c.Do(req, withRequiredProps(transport.PropertiesReliable))
 	if err != nil {
 		return nil, err
 	}
@@ -117,7 +118,9 @@ func (c *Client) sendAuthRequest(ctx context.Context, cfg *authConfig, username,
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Auth0-Client", "eyJuYW1lIjoiYXV0aDAuanMtdWxwIiwidmVyc2lvbiI6IjkuMjQuMSJ9")
 
-	resp, err := c.Do(req)
+	resp, err := c.Do(req,
+		withRequiredProps(transport.PropertiesReliable),
+		withExpectedStatuses(http.StatusOK, http.StatusUnauthorized))
 	if err != nil {
 		return "", "", err
 	}
@@ -156,7 +159,7 @@ func (c *Client) sendPostAuth(ctx context.Context, token, params string) (*http.
 
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
-	resp, err := c.Do(req)
+	resp, err := c.Do(req, withRequiredProps(transport.PropertiesReliable))
 	if err != nil {
 		return nil, err
 	}
diff --git a/backend/internal/ibd/auth_test.go b/backend/internal/ibd/auth_test.go
index 8a00d42..54ea98a 100644
--- a/backend/internal/ibd/auth_test.go
+++ b/backend/internal/ibd/auth_test.go
@@ -163,9 +163,7 @@ func TestClient_Authenticate(t *testing.T) {
 			return resp, nil
 		})
 
-	client := NewClient([]transport.Transport{
-		&http.Client{Transport: tp},
-	}, nil)
+	client := NewClient(nil, newTransport(tp))
 
 	cookie, err := client.Authenticate(context.Background(), "abc", "xyz")
 	require.NoError(t, err)
@@ -191,11 +189,27 @@ func TestClient_Authenticate_401(t *testing.T) {
 			return httpmock.NewStringResponse(http.StatusUnauthorized, `{"name":"ValidationError","code":"ERR016","message":"Wrong username or password","description":"Wrong username or password"}`), nil
 		})
 
-	client := NewClient([]transport.Transport{
-		&http.Client{Transport: tp},
-	}, nil)
+	client := NewClient(nil, newTransport(tp))
 
 	cookie, err := client.Authenticate(context.Background(), "abc", "xyz")
 	assert.Nil(t, cookie)
 	assert.ErrorIs(t, err, ErrBadCredentials)
 }
+
+type testReliableTransport http.Client
+
+func newTransport(tp *httpmock.MockTransport) *testReliableTransport {
+	return (*testReliableTransport)(&http.Client{Transport: tp})
+}
+
+func (t *testReliableTransport) String() string {
+	return "testReliableTransport"
+}
+
+func (t *testReliableTransport) Do(req *http.Request) (*http.Response, error) {
+	return (*http.Client)(t).Do(req)
+}
+
+func (t *testReliableTransport) Properties() transport.Properties {
+	return transport.PropertiesFree | transport.PropertiesReliable
+}
diff --git a/backend/internal/ibd/check_ibd_username.go b/backend/internal/ibd/check_ibd_username.go
index c03176e..b026151 100644
--- a/backend/internal/ibd/check_ibd_username.go
+++ b/backend/internal/ibd/check_ibd_username.go
@@ -42,7 +42,7 @@ func (c *Client) checkIBDUsername(ctx context.Context, cfg *authConfig, username
 	req.Header.Set("X-REQUEST-EDITIONID", "IBD-EN_US")
 	req.Header.Set("X-REQUEST-SCHEME", "https")
 
-	resp, err := c.DoWithStatus(req, []int{http.StatusOK, http.StatusUnauthorized})
+	resp, err := c.Do(req, withExpectedStatuses(http.StatusOK, http.StatusUnauthorized))
 	if err != nil {
 		return false, err
 	}
diff --git a/backend/internal/ibd/client.go b/backend/internal/ibd/client.go
index 2b91268..25c5173 100644
--- a/backend/internal/ibd/client.go
+++ b/backend/internal/ibd/client.go
@@ -20,8 +20,8 @@ type Client struct {
 }
 
 func NewClient(
-	transports []transport.Transport,
 	cookies database.CookieSource,
+	transports ...transport.Transport,
 ) *Client {
 	return &Client{transports, cookies}
 }
@@ -55,12 +55,17 @@ func (c *Client) getCookie(ctx context.Context, subject *string) (uint, *http.Co
 	return cookie.ID, cookie.ToHTTPCookie(), nil
 }
 
-func (c *Client) Do(req *http.Request) (*http.Response, error) {
-	return c.DoWithStatus(req, []int{http.StatusOK})
-}
+func (c *Client) Do(req *http.Request, opts ...optionFunc) (*http.Response, error) {
+	o := defaultOptions
+	for _, opt := range opts {
+		opt(&o)
+	}
+
+	// Sort and filter transports by properties
+	transports := transport.FilterTransports(c.transports, o.requiredProps)
+	transport.SortTransports(transports)
 
-func (c *Client) DoWithStatus(req *http.Request, expectedStatus []int) (*http.Response, error) {
-	for i, tp := range c.transports {
+	for _, tp := range transports {
 		resp, err := tp.Do(req)
 		if errors.Is(err, transport.ErrUnsupportedRequest) {
 			// Skip unsupported transport
@@ -68,17 +73,17 @@ func (c *Client) DoWithStatus(req *http.Request, expectedStatus []int) (*http.Re
 		}
 		if err != nil {
 			slog.ErrorContext(req.Context(), "transport error",
-				"transport", i,
+				"transport", tp.String(),
 				"error", err,
 			)
 			continue
 		}
-		if slices.Contains(expectedStatus, resp.StatusCode) {
+		if slices.Contains(o.expectedStatuses, resp.StatusCode) {
 			return resp, nil
 		} else {
 			slog.ErrorContext(req.Context(), "unexpected status code",
-				"transport", i,
-				"expected", expectedStatus,
+				"transport", tp.String(),
+				"expected", o.expectedStatuses,
 				"actual", resp.StatusCode,
 			)
 			continue
diff --git a/backend/internal/ibd/client_test.go b/backend/internal/ibd/client_test.go
index 0a8fa98..d2dc1b2 100644
--- a/backend/internal/ibd/client_test.go
+++ b/backend/internal/ibd/client_test.go
@@ -16,10 +16,7 @@ func TestClient_getCookie(t *testing.T) {
 	t.Run("no cookies", func(t *testing.T) {
 		t.Parallel()
 
-		client := NewClient(
-			nil,
-			new(emptyCookieSourceStub),
-		)
+		client := NewClient(new(emptyCookieSourceStub))
 
 		_, _, err := client.getCookie(context.Background(), nil)
 		assert.ErrorIs(t, err, ErrNoAvailableCookies)
@@ -28,10 +25,7 @@ func TestClient_getCookie(t *testing.T) {
 	t.Run("no cookies by subject", func(t *testing.T) {
 		t.Parallel()
 
-		client := NewClient(
-			nil,
-			new(emptyCookieSourceStub),
-		)
+		client := NewClient(new(emptyCookieSourceStub))
 
 		subject := "test"
 		_, _, err := client.getCookie(context.Background(), &subject)
@@ -41,10 +35,7 @@ func TestClient_getCookie(t *testing.T) {
 	t.Run("get any cookie", func(t *testing.T) {
 		t.Parallel()
 
-		client := NewClient(
-			nil,
-			new(emptyCookieSourceStub),
-		)
+		client := NewClient(new(cookieSourceStub))
 
 		id, cookie, err := client.getCookie(context.Background(), nil)
 		require.NoError(t, err)
@@ -59,10 +50,7 @@ func TestClient_getCookie(t *testing.T) {
 	t.Run("get cookie by subject", func(t *testing.T) {
 		t.Parallel()
 
-		client := NewClient(
-			nil,
-			new(emptyCookieSourceStub),
-		)
+		client := NewClient(new(cookieSourceStub))
 
 		subject := "test"
 		id, cookie, err := client.getCookie(context.Background(), &subject)
diff --git a/backend/internal/ibd/options.go b/backend/internal/ibd/options.go
new file mode 100644
index 0000000..5c378d5
--- /dev/null
+++ b/backend/internal/ibd/options.go
@@ -0,0 +1,26 @@
+package ibd
+
+import "github.com/ansg191/ibd-trader-backend/internal/ibd/transport"
+
+type optionFunc func(*options)
+
+var defaultOptions = options{
+	expectedStatuses: []int{200},
+}
+
+type options struct {
+	expectedStatuses []int
+	requiredProps    transport.Properties
+}
+
+func withExpectedStatuses(statuses ...int) optionFunc {
+	return func(o *options) {
+		o.expectedStatuses = append(o.expectedStatuses, statuses...)
+	}
+}
+
+func withRequiredProps(props transport.Properties) optionFunc {
+	return func(o *options) {
+		o.requiredProps = props
+	}
+}
diff --git a/backend/internal/ibd/search_test.go b/backend/internal/ibd/search_test.go
index f291033..99157cf 100644
--- a/backend/internal/ibd/search_test.go
+++ b/backend/internal/ibd/search_test.go
@@ -195,9 +195,7 @@ func TestClient_Search(t *testing.T) {
 			tp := httpmock.NewMockTransport()
 			tp.RegisterResponder("GET", searchUrl, httpmock.NewStringResponder(200, tt.response))
 
-			client := NewClient([]transport.Transport{
-				&http.Client{Transport: tp},
-			}, new(cookieSourceStub))
+			client := NewClient(new(cookieSourceStub), transport.NewStandardTransport(&http.Client{Transport: tp}))
 
 			tt.f(t, client)
 		})
diff --git a/backend/internal/ibd/transport/scrapfly/scrapfly.go b/backend/internal/ibd/transport/scrapfly/scrapfly.go
index f34f3aa..3b414de 100644
--- a/backend/internal/ibd/transport/scrapfly/scrapfly.go
+++ b/backend/internal/ibd/transport/scrapfly/scrapfly.go
@@ -30,6 +30,10 @@ func New(client *http.Client, apiKey string, opts ...ScrapeOption) *ScrapflyTran
 	}
 }
 
+func (s *ScrapflyTransport) String() string {
+	return "scrapfly"
+}
+
 func (s *ScrapflyTransport) Do(req *http.Request) (*http.Response, error) {
 	// Construct scrape request URL
 	scrapeUrl, err := url.Parse(s.options.baseURL)
@@ -70,6 +74,10 @@ func (s *ScrapflyTransport) Do(req *http.Request) (*http.Response, error) {
 	return scraperResponse.ToHTTPResponse()
 }
 
+func (s *ScrapflyTransport) Properties() transport.Properties {
+	return transport.PropertiesReliable
+}
+
 func (s *ScrapflyTransport) constructRawQuery(u *url.URL, headers http.Header) string {
 	params := url.Values{}
 	params.Set("key", s.apiKey)
diff --git a/backend/internal/ibd/transport/standard.go b/backend/internal/ibd/transport/standard.go
new file mode 100644
index 0000000..9fa9ff9
--- /dev/null
+++ b/backend/internal/ibd/transport/standard.go
@@ -0,0 +1,41 @@
+package transport
+
+import (
+	"net/http"
+
+	"github.com/EDDYCJY/fake-useragent"
+)
+
+type StandardTransport http.Client
+
+func NewStandardTransport(client *http.Client) *StandardTransport {
+	return (*StandardTransport)(client)
+}
+
+func (t *StandardTransport) Do(req *http.Request) (*http.Response, error) {
+	addFakeHeaders(req)
+	return (*http.Client)(t).Do(req)
+}
+
+func (t *StandardTransport) String() string {
+	return "standard"
+}
+
+func (t *StandardTransport) Properties() Properties {
+	return PropertiesFree
+}
+
+func addFakeHeaders(req *http.Request) {
+	req.Header.Set("User-Agent", browser.Linux())
+	req.Header.Set("Sec-CH-UA", `"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"`)
+	req.Header.Set("Sec-CH-UA-Mobile", "?0")
+	req.Header.Set("Sec-CH-UA-Platform", "Linux")
+	req.Header.Set("Upgrade-Insecure-Requests", "1")
+	req.Header.Set("Priority", "u=0, i")
+	req.Header.Set("Sec-Fetch-Site", "none")
+	req.Header.Set("Sec-Fetch-Mode", "navigate")
+	req.Header.Set("Sec-Fetch-Dest", "document")
+	req.Header.Set("Sec-Fetch-User", "?1")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
+}
diff --git a/backend/internal/ibd/transport/transport.go b/backend/internal/ibd/transport/transport.go
index 7918f4f..95e9ef3 100644
--- a/backend/internal/ibd/transport/transport.go
+++ b/backend/internal/ibd/transport/transport.go
@@ -1,12 +1,66 @@
 package transport
 
 import (
+	"cmp"
 	"errors"
+	"fmt"
 	"net/http"
+	"slices"
 )
 
 var ErrUnsupportedRequest = errors.New("unsupported request")
 
+type Properties uint8
+
+const (
+	// PropertiesFree indicates that the transport is free.
+	// This means that requests made with this transport don't cost any money.
+	PropertiesFree Properties = 1 << iota
+	// PropertiesReliable indicates that the transport is reliable.
+	// This means that requests made with this transport are guaranteed to be
+	// successful if the server is reachable.
+	PropertiesReliable
+)
+
+func (p Properties) IsReliable() bool {
+	return p&PropertiesReliable != 0
+}
+
+func (p Properties) IsFree() bool {
+	return p&PropertiesFree != 0
+}
+
 type Transport interface {
+	fmt.Stringer
+
 	Do(req *http.Request) (*http.Response, error)
+	Properties() Properties
+}
+
+// SortTransports sorts the transports by their properties.
+//
+// The transports are sorted in the following order:
+// 1. Free transports
+// 2. Reliable transports
+func SortTransports(transports []Transport) {
+	priorities := map[Properties]int{
+		PropertiesFree | PropertiesReliable: 0,
+		PropertiesFree:                      1,
+		PropertiesReliable:                  2,
+	}
+	slices.SortStableFunc(transports, func(a, b Transport) int {
+		iPriority := priorities[a.Properties()]
+		jPriority := priorities[b.Properties()]
+		return cmp.Compare(iPriority, jPriority)
+	})
+}
+
+func FilterTransports(transport []Transport, props Properties) []Transport {
+	var filtered []Transport
+	for _, tp := range transport {
+		if tp.Properties()&props == props {
+			filtered = append(filtered, tp)
+		}
+	}
+	return filtered
 }
diff --git a/backend/internal/server/idb/user/v1/user.go b/backend/internal/server/idb/user/v1/user.go
index 8e5f16a..c100465 100644
--- a/backend/internal/server/idb/user/v1/user.go
+++ b/backend/internal/server/idb/user/v1/user.go
@@ -17,21 +17,26 @@ import (
 type Server struct {
 	pb.UnimplementedUserServiceServer
 
-	db     database.UserStore
+	user   database.UserStore
+	cookie database.CookieSource
 	client *ibd.Client
 }
 
-func New(db database.UserStore, client *ibd.Client) *Server {
-	return &Server{db: db, client: client}
+func New(userStore database.UserStore, cookieStore database.CookieStore, client *ibd.Client) *Server {
+	return &Server{
+		user:   userStore,
+		cookie: cookieStore,
+		client: client,
+	}
 }
 
 func (u *Server) CreateUser(ctx context.Context, request *pb.CreateUserRequest) (*pb.CreateUserResponse, error) {
-	err := u.db.AddUser(ctx, request.Subject)
+	err := u.user.AddUser(ctx, request.Subject)
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "unable to create user: %v", err)
 	}
 
-	user, err := u.db.GetUser(ctx, request.Subject)
+	user, err := u.user.GetUser(ctx, request.Subject)
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "unable to get user: %v", err)
 	}
@@ -46,7 +51,7 @@ func (u *Server) CreateUser(ctx context.Context, request *pb.CreateUserRequest)
 }
 
 func (u *Server) GetUser(ctx context.Context, request *pb.GetUserRequest) (*pb.GetUserResponse, error) {
-	user, err := u.db.GetUser(ctx, request.Subject)
+	user, err := u.user.GetUser(ctx, request.Subject)
 	if errors.Is(err, database.ErrUserNotFound) {
 		return nil, status.New(codes.NotFound, "user not found").Err()
 	}
@@ -83,7 +88,7 @@ func (u *Server) UpdateUser(ctx context.Context, request *pb.UpdateUserRequest)
 		(newUser.IbdPassword != existingUser.IbdPassword ||
 			newUser.IbdUsername != existingUser.IbdUsername) {
 		// Update IBD creds
-		err = u.db.AddIBDCreds(ctx, newUser.Subject, *newUser.IbdUsername, *newUser.IbdPassword)
+		err = u.user.AddIBDCreds(ctx, newUser.Subject, *newUser.IbdUsername, *newUser.IbdPassword)
 		if err != nil {
 			return nil, status.Errorf(codes.Internal, "unable to update user: %v", err)
 		}
@@ -111,3 +116,41 @@ func (u *Server) CheckIBDUsername(ctx context.Context, req *pb.CheckIBDUsernameR
 		Exists: exists,
 	}, nil
 }
+
+func (u *Server) AuthenticateUser(ctx context.Context, req *pb.AuthenticateUserRequest) (*pb.AuthenticateUserResponse, error) {
+	// Check if user has cookies
+	cookies, err := u.cookie.GetCookies(ctx, req.Subject, false)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "unable to get cookies: %v", err)
+	}
+	if len(cookies) > 0 {
+		return &pb.AuthenticateUserResponse{
+			Authenticated: true,
+		}, nil
+	}
+
+	// Authenticate user
+	// Get IBD creds
+	username, password, err := u.user.GetIBDCreds(ctx, req.Subject)
+	if errors.Is(err, database.ErrIBDCredsNotFound) {
+		return nil, status.New(codes.NotFound, "User has no IDB creds").Err()
+	}
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "unable to get IBD creds: %v", err)
+	}
+
+	// Authenticate user
+	cookie, err := u.client.Authenticate(ctx, username, password)
+	if errors.Is(err, ibd.ErrBadCredentials) {
+		return &pb.AuthenticateUserResponse{
+			Authenticated: false,
+		}, nil
+	}
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "unable to authenticate user: %v", err)
+	}
+
+	return &pb.AuthenticateUserResponse{
+		Authenticated: cookie != nil,
+	}, nil
+}
diff --git a/backend/internal/server/server.go b/backend/internal/server/server.go
index c46a629..186d581 100644
--- a/backend/internal/server/server.go
+++ b/backend/internal/server/server.go
@@ -45,7 +45,7 @@ func New(
 	}
 
 	s := grpc.NewServer()
-	upb.RegisterUserServiceServer(s, user.New(db, client))
+	upb.RegisterUserServiceServer(s, user.New(db, db, client))
 	spb.RegisterStockServiceServer(s, stock.New(db, scrapeQueue))
 	longrunningpb.RegisterOperationsServer(s, newOperationServer(scrapeQueue))
 	reflection.Register(s)