aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
m---------backend/api0
-rw-r--r--backend/cmd/main.go6
-rw-r--r--backend/go.mod3
-rw-r--r--backend/go.sum16
-rw-r--r--backend/internal/ibd/auth.go9
-rw-r--r--backend/internal/ibd/auth_test.go26
-rw-r--r--backend/internal/ibd/check_ibd_username.go2
-rw-r--r--backend/internal/ibd/client.go25
-rw-r--r--backend/internal/ibd/client_test.go20
-rw-r--r--backend/internal/ibd/options.go26
-rw-r--r--backend/internal/ibd/search_test.go4
-rw-r--r--backend/internal/ibd/transport/scrapfly/scrapfly.go8
-rw-r--r--backend/internal/ibd/transport/standard.go41
-rw-r--r--backend/internal/ibd/transport/transport.go54
-rw-r--r--backend/internal/server/idb/user/v1/user.go57
-rw-r--r--backend/internal/server/server.go2
16 files changed, 249 insertions, 50 deletions
diff --git a/backend/api b/backend/api
-Subproject 75ae31c95e6cbba285785e6029c9de93ff6a47b
+Subproject 250159ea00457090d9cc7b3cc270618d14ce9a9
diff --git a/backend/cmd/main.go b/backend/cmd/main.go
index 16fc1ce..6b49980 100644
--- a/backend/cmd/main.go
+++ b/backend/cmd/main.go
@@ -154,10 +154,10 @@ func setupIBDClient(cfg *config.Config, db database.Database) (*ibd.Client, erro
t := http.DefaultTransport.(*http.Transport).Clone()
t.Proxy = http.ProxyURL(pUrl)
transports := []transport.Transport{
- &http.Client{Transport: t}, // Default proxied transport
- scrapfly.New(http.DefaultClient, cfg.IBD.APIKey), // Scrapfly transport
+ transport.NewStandardTransport(&http.Client{Transport: t}), // Default proxied transport
+ scrapfly.New(http.DefaultClient, cfg.IBD.APIKey), // Scrapfly transport
}
- client := ibd.NewClient(transports, db)
+ client := ibd.NewClient(db, transports...)
return client, nil
}
diff --git a/backend/go.mod b/backend/go.mod
index bd55e54..cdd1e1c 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -7,6 +7,7 @@ toolchain go1.22.5
require (
cloud.google.com/go/kms v1.18.4
cloud.google.com/go/longrunning v0.5.11
+ github.com/EDDYCJY/fake-useragent v0.2.0
github.com/Rhymond/go-money v1.0.14
github.com/bsm/redislock v0.9.4
github.com/buraksezer/consistent v0.10.0
@@ -37,6 +38,8 @@ require (
cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
cloud.google.com/go/iam v1.1.12 // indirect
+ github.com/PuerkitoBio/goquery v1.9.2 // indirect
+ github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index 3f5e220..d7235e3 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -395,11 +395,17 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/EDDYCJY/fake-useragent v0.2.0 h1:Jcnkk2bgXmDpX0z+ELlUErTkoLb/mxFBNd2YdcpvJBs=
+github.com/EDDYCJY/fake-useragent v0.2.0/go.mod h1:5wn3zzlDxhKW6NYknushqinPcAqZcAPHy8lLczCdJdc=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
+github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/Rhymond/go-money v1.0.14 h1:HtdIZ0mP4LrnpN3wdRhsik7pool7x22ILZdDe3moL6E=
github.com/Rhymond/go-money v1.0.14/go.mod h1:iHvCuIvitxu2JIlAlhF0g9jHqjRSr+rpdOs7Omqlupg=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
@@ -746,6 +752,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -795,6 +802,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -903,10 +912,14 @@ golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -918,6 +931,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -979,6 +994,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/backend/internal/ibd/auth.go b/backend/internal/ibd/auth.go
index f09f3f7..7b82057 100644
--- a/backend/internal/ibd/auth.go
+++ b/backend/internal/ibd/auth.go
@@ -11,6 +11,7 @@ import (
"net/http"
"strings"
+ "github.com/ansg191/ibd-trader-backend/internal/ibd/transport"
"golang.org/x/net/html"
)
@@ -48,7 +49,7 @@ func (c *Client) getLoginPage(ctx context.Context) (*authConfig, error) {
return nil, err
}
- resp, err := c.Do(req)
+ resp, err := c.Do(req, withRequiredProps(transport.PropertiesReliable))
if err != nil {
return nil, err
}
@@ -117,7 +118,9 @@ func (c *Client) sendAuthRequest(ctx context.Context, cfg *authConfig, username,
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Auth0-Client", "eyJuYW1lIjoiYXV0aDAuanMtdWxwIiwidmVyc2lvbiI6IjkuMjQuMSJ9")
- resp, err := c.Do(req)
+ resp, err := c.Do(req,
+ withRequiredProps(transport.PropertiesReliable),
+ withExpectedStatuses(http.StatusOK, http.StatusUnauthorized))
if err != nil {
return "", "", err
}
@@ -156,7 +159,7 @@ func (c *Client) sendPostAuth(ctx context.Context, token, params string) (*http.
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- resp, err := c.Do(req)
+ resp, err := c.Do(req, withRequiredProps(transport.PropertiesReliable))
if err != nil {
return nil, err
}
diff --git a/backend/internal/ibd/auth_test.go b/backend/internal/ibd/auth_test.go
index 8a00d42..54ea98a 100644
--- a/backend/internal/ibd/auth_test.go
+++ b/backend/internal/ibd/auth_test.go
@@ -163,9 +163,7 @@ func TestClient_Authenticate(t *testing.T) {
return resp, nil
})
- client := NewClient([]transport.Transport{
- &http.Client{Transport: tp},
- }, nil)
+ client := NewClient(nil, newTransport(tp))
cookie, err := client.Authenticate(context.Background(), "abc", "xyz")
require.NoError(t, err)
@@ -191,11 +189,27 @@ func TestClient_Authenticate_401(t *testing.T) {
return httpmock.NewStringResponse(http.StatusUnauthorized, `{"name":"ValidationError","code":"ERR016","message":"Wrong username or password","description":"Wrong username or password"}`), nil
})
- client := NewClient([]transport.Transport{
- &http.Client{Transport: tp},
- }, nil)
+ client := NewClient(nil, newTransport(tp))
cookie, err := client.Authenticate(context.Background(), "abc", "xyz")
assert.Nil(t, cookie)
assert.ErrorIs(t, err, ErrBadCredentials)
}
+
+type testReliableTransport http.Client
+
+func newTransport(tp *httpmock.MockTransport) *testReliableTransport {
+ return (*testReliableTransport)(&http.Client{Transport: tp})
+}
+
+func (t *testReliableTransport) String() string {
+ return "testReliableTransport"
+}
+
+func (t *testReliableTransport) Do(req *http.Request) (*http.Response, error) {
+ return (*http.Client)(t).Do(req)
+}
+
+func (t *testReliableTransport) Properties() transport.Properties {
+ return transport.PropertiesFree | transport.PropertiesReliable
+}
diff --git a/backend/internal/ibd/check_ibd_username.go b/backend/internal/ibd/check_ibd_username.go
index c03176e..b026151 100644
--- a/backend/internal/ibd/check_ibd_username.go
+++ b/backend/internal/ibd/check_ibd_username.go
@@ -42,7 +42,7 @@ func (c *Client) checkIBDUsername(ctx context.Context, cfg *authConfig, username
req.Header.Set("X-REQUEST-EDITIONID", "IBD-EN_US")
req.Header.Set("X-REQUEST-SCHEME", "https")
- resp, err := c.DoWithStatus(req, []int{http.StatusOK, http.StatusUnauthorized})
+ resp, err := c.Do(req, withExpectedStatuses(http.StatusOK, http.StatusUnauthorized))
if err != nil {
return false, err
}
diff --git a/backend/internal/ibd/client.go b/backend/internal/ibd/client.go
index 2b91268..25c5173 100644
--- a/backend/internal/ibd/client.go
+++ b/backend/internal/ibd/client.go
@@ -20,8 +20,8 @@ type Client struct {
}
func NewClient(
- transports []transport.Transport,
cookies database.CookieSource,
+ transports ...transport.Transport,
) *Client {
return &Client{transports, cookies}
}
@@ -55,12 +55,17 @@ func (c *Client) getCookie(ctx context.Context, subject *string) (uint, *http.Co
return cookie.ID, cookie.ToHTTPCookie(), nil
}
-func (c *Client) Do(req *http.Request) (*http.Response, error) {
- return c.DoWithStatus(req, []int{http.StatusOK})
-}
+func (c *Client) Do(req *http.Request, opts ...optionFunc) (*http.Response, error) {
+ o := defaultOptions
+ for _, opt := range opts {
+ opt(&o)
+ }
+
+ // Sort and filter transports by properties
+ transports := transport.FilterTransports(c.transports, o.requiredProps)
+ transport.SortTransports(transports)
-func (c *Client) DoWithStatus(req *http.Request, expectedStatus []int) (*http.Response, error) {
- for i, tp := range c.transports {
+ for _, tp := range transports {
resp, err := tp.Do(req)
if errors.Is(err, transport.ErrUnsupportedRequest) {
// Skip unsupported transport
@@ -68,17 +73,17 @@ func (c *Client) DoWithStatus(req *http.Request, expectedStatus []int) (*http.Re
}
if err != nil {
slog.ErrorContext(req.Context(), "transport error",
- "transport", i,
+ "transport", tp.String(),
"error", err,
)
continue
}
- if slices.Contains(expectedStatus, resp.StatusCode) {
+ if slices.Contains(o.expectedStatuses, resp.StatusCode) {
return resp, nil
} else {
slog.ErrorContext(req.Context(), "unexpected status code",
- "transport", i,
- "expected", expectedStatus,
+ "transport", tp.String(),
+ "expected", o.expectedStatuses,
"actual", resp.StatusCode,
)
continue
diff --git a/backend/internal/ibd/client_test.go b/backend/internal/ibd/client_test.go
index 0a8fa98..d2dc1b2 100644
--- a/backend/internal/ibd/client_test.go
+++ b/backend/internal/ibd/client_test.go
@@ -16,10 +16,7 @@ func TestClient_getCookie(t *testing.T) {
t.Run("no cookies", func(t *testing.T) {
t.Parallel()
- client := NewClient(
- nil,
- new(emptyCookieSourceStub),
- )
+ client := NewClient(new(emptyCookieSourceStub))
_, _, err := client.getCookie(context.Background(), nil)
assert.ErrorIs(t, err, ErrNoAvailableCookies)
@@ -28,10 +25,7 @@ func TestClient_getCookie(t *testing.T) {
t.Run("no cookies by subject", func(t *testing.T) {
t.Parallel()
- client := NewClient(
- nil,
- new(emptyCookieSourceStub),
- )
+ client := NewClient(new(emptyCookieSourceStub))
subject := "test"
_, _, err := client.getCookie(context.Background(), &subject)
@@ -41,10 +35,7 @@ func TestClient_getCookie(t *testing.T) {
t.Run("get any cookie", func(t *testing.T) {
t.Parallel()
- client := NewClient(
- nil,
- new(emptyCookieSourceStub),
- )
+ client := NewClient(new(cookieSourceStub))
id, cookie, err := client.getCookie(context.Background(), nil)
require.NoError(t, err)
@@ -59,10 +50,7 @@ func TestClient_getCookie(t *testing.T) {
t.Run("get cookie by subject", func(t *testing.T) {
t.Parallel()
- client := NewClient(
- nil,
- new(emptyCookieSourceStub),
- )
+ client := NewClient(new(cookieSourceStub))
subject := "test"
id, cookie, err := client.getCookie(context.Background(), &subject)
diff --git a/backend/internal/ibd/options.go b/backend/internal/ibd/options.go
new file mode 100644
index 0000000..5c378d5
--- /dev/null
+++ b/backend/internal/ibd/options.go
@@ -0,0 +1,26 @@
+package ibd
+
+import "github.com/ansg191/ibd-trader-backend/internal/ibd/transport"
+
+type optionFunc func(*options)
+
+var defaultOptions = options{
+ expectedStatuses: []int{200},
+}
+
+type options struct {
+ expectedStatuses []int
+ requiredProps transport.Properties
+}
+
+func withExpectedStatuses(statuses ...int) optionFunc {
+ return func(o *options) {
+ o.expectedStatuses = append(o.expectedStatuses, statuses...)
+ }
+}
+
+func withRequiredProps(props transport.Properties) optionFunc {
+ return func(o *options) {
+ o.requiredProps = props
+ }
+}
diff --git a/backend/internal/ibd/search_test.go b/backend/internal/ibd/search_test.go
index f291033..99157cf 100644
--- a/backend/internal/ibd/search_test.go
+++ b/backend/internal/ibd/search_test.go
@@ -195,9 +195,7 @@ func TestClient_Search(t *testing.T) {
tp := httpmock.NewMockTransport()
tp.RegisterResponder("GET", searchUrl, httpmock.NewStringResponder(200, tt.response))
- client := NewClient([]transport.Transport{
- &http.Client{Transport: tp},
- }, new(cookieSourceStub))
+ client := NewClient(new(cookieSourceStub), transport.NewStandardTransport(&http.Client{Transport: tp}))
tt.f(t, client)
})
diff --git a/backend/internal/ibd/transport/scrapfly/scrapfly.go b/backend/internal/ibd/transport/scrapfly/scrapfly.go
index f34f3aa..3b414de 100644
--- a/backend/internal/ibd/transport/scrapfly/scrapfly.go
+++ b/backend/internal/ibd/transport/scrapfly/scrapfly.go
@@ -30,6 +30,10 @@ func New(client *http.Client, apiKey string, opts ...ScrapeOption) *ScrapflyTran
}
}
+func (s *ScrapflyTransport) String() string {
+ return "scrapfly"
+}
+
func (s *ScrapflyTransport) Do(req *http.Request) (*http.Response, error) {
// Construct scrape request URL
scrapeUrl, err := url.Parse(s.options.baseURL)
@@ -70,6 +74,10 @@ func (s *ScrapflyTransport) Do(req *http.Request) (*http.Response, error) {
return scraperResponse.ToHTTPResponse()
}
+func (s *ScrapflyTransport) Properties() transport.Properties {
+ return transport.PropertiesReliable
+}
+
func (s *ScrapflyTransport) constructRawQuery(u *url.URL, headers http.Header) string {
params := url.Values{}
params.Set("key", s.apiKey)
diff --git a/backend/internal/ibd/transport/standard.go b/backend/internal/ibd/transport/standard.go
new file mode 100644
index 0000000..9fa9ff9
--- /dev/null
+++ b/backend/internal/ibd/transport/standard.go
@@ -0,0 +1,41 @@
+package transport
+
+import (
+ "net/http"
+
+ "github.com/EDDYCJY/fake-useragent"
+)
+
+type StandardTransport http.Client
+
+func NewStandardTransport(client *http.Client) *StandardTransport {
+ return (*StandardTransport)(client)
+}
+
+func (t *StandardTransport) Do(req *http.Request) (*http.Response, error) {
+ addFakeHeaders(req)
+ return (*http.Client)(t).Do(req)
+}
+
+func (t *StandardTransport) String() string {
+ return "standard"
+}
+
+func (t *StandardTransport) Properties() Properties {
+ return PropertiesFree
+}
+
+func addFakeHeaders(req *http.Request) {
+ req.Header.Set("User-Agent", browser.Linux())
+ req.Header.Set("Sec-CH-UA", `"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"`)
+ req.Header.Set("Sec-CH-UA-Mobile", "?0")
+ req.Header.Set("Sec-CH-UA-Platform", "Linux")
+ req.Header.Set("Upgrade-Insecure-Requests", "1")
+ req.Header.Set("Priority", "u=0, i")
+ req.Header.Set("Sec-Fetch-Site", "none")
+ req.Header.Set("Sec-Fetch-Mode", "navigate")
+ req.Header.Set("Sec-Fetch-Dest", "document")
+ req.Header.Set("Sec-Fetch-User", "?1")
+ req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+ req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
+}
diff --git a/backend/internal/ibd/transport/transport.go b/backend/internal/ibd/transport/transport.go
index 7918f4f..95e9ef3 100644
--- a/backend/internal/ibd/transport/transport.go
+++ b/backend/internal/ibd/transport/transport.go
@@ -1,12 +1,66 @@
package transport
import (
+ "cmp"
"errors"
+ "fmt"
"net/http"
+ "slices"
)
var ErrUnsupportedRequest = errors.New("unsupported request")
+type Properties uint8
+
+const (
+ // PropertiesFree indicates that the transport is free.
+ // This means that requests made with this transport don't cost any money.
+ PropertiesFree Properties = 1 << iota
+ // PropertiesReliable indicates that the transport is reliable.
+ // This means that requests made with this transport are guaranteed to be
+ // successful if the server is reachable.
+ PropertiesReliable
+)
+
+func (p Properties) IsReliable() bool {
+ return p&PropertiesReliable != 0
+}
+
+func (p Properties) IsFree() bool {
+ return p&PropertiesFree != 0
+}
+
type Transport interface {
+ fmt.Stringer
+
Do(req *http.Request) (*http.Response, error)
+ Properties() Properties
+}
+
+// SortTransports sorts the transports by their properties.
+//
+// The transports are sorted in the following order:
+// 1. Free transports
+// 2. Reliable transports
+func SortTransports(transports []Transport) {
+ priorities := map[Properties]int{
+ PropertiesFree | PropertiesReliable: 0,
+ PropertiesFree: 1,
+ PropertiesReliable: 2,
+ }
+ slices.SortStableFunc(transports, func(a, b Transport) int {
+ iPriority := priorities[a.Properties()]
+ jPriority := priorities[b.Properties()]
+ return cmp.Compare(iPriority, jPriority)
+ })
+}
+
+func FilterTransports(transport []Transport, props Properties) []Transport {
+ var filtered []Transport
+ for _, tp := range transport {
+ if tp.Properties()&props == props {
+ filtered = append(filtered, tp)
+ }
+ }
+ return filtered
}
diff --git a/backend/internal/server/idb/user/v1/user.go b/backend/internal/server/idb/user/v1/user.go
index 8e5f16a..c100465 100644
--- a/backend/internal/server/idb/user/v1/user.go
+++ b/backend/internal/server/idb/user/v1/user.go
@@ -17,21 +17,26 @@ import (
type Server struct {
pb.UnimplementedUserServiceServer
- db database.UserStore
+ user database.UserStore
+ cookie database.CookieSource
client *ibd.Client
}
-func New(db database.UserStore, client *ibd.Client) *Server {
- return &Server{db: db, client: client}
+func New(userStore database.UserStore, cookieStore database.CookieStore, client *ibd.Client) *Server {
+ return &Server{
+ user: userStore,
+ cookie: cookieStore,
+ client: client,
+ }
}
func (u *Server) CreateUser(ctx context.Context, request *pb.CreateUserRequest) (*pb.CreateUserResponse, error) {
- err := u.db.AddUser(ctx, request.Subject)
+ err := u.user.AddUser(ctx, request.Subject)
if err != nil {
return nil, status.Errorf(codes.Internal, "unable to create user: %v", err)
}
- user, err := u.db.GetUser(ctx, request.Subject)
+ user, err := u.user.GetUser(ctx, request.Subject)
if err != nil {
return nil, status.Errorf(codes.Internal, "unable to get user: %v", err)
}
@@ -46,7 +51,7 @@ func (u *Server) CreateUser(ctx context.Context, request *pb.CreateUserRequest)
}
func (u *Server) GetUser(ctx context.Context, request *pb.GetUserRequest) (*pb.GetUserResponse, error) {
- user, err := u.db.GetUser(ctx, request.Subject)
+ user, err := u.user.GetUser(ctx, request.Subject)
if errors.Is(err, database.ErrUserNotFound) {
return nil, status.New(codes.NotFound, "user not found").Err()
}
@@ -83,7 +88,7 @@ func (u *Server) UpdateUser(ctx context.Context, request *pb.UpdateUserRequest)
(newUser.IbdPassword != existingUser.IbdPassword ||
newUser.IbdUsername != existingUser.IbdUsername) {
// Update IBD creds
- err = u.db.AddIBDCreds(ctx, newUser.Subject, *newUser.IbdUsername, *newUser.IbdPassword)
+ err = u.user.AddIBDCreds(ctx, newUser.Subject, *newUser.IbdUsername, *newUser.IbdPassword)
if err != nil {
return nil, status.Errorf(codes.Internal, "unable to update user: %v", err)
}
@@ -111,3 +116,41 @@ func (u *Server) CheckIBDUsername(ctx context.Context, req *pb.CheckIBDUsernameR
Exists: exists,
}, nil
}
+
+func (u *Server) AuthenticateUser(ctx context.Context, req *pb.AuthenticateUserRequest) (*pb.AuthenticateUserResponse, error) {
+ // Check if user has cookies
+ cookies, err := u.cookie.GetCookies(ctx, req.Subject, false)
+ if err != nil {
+ return nil, status.Errorf(codes.Internal, "unable to get cookies: %v", err)
+ }
+ if len(cookies) > 0 {
+ return &pb.AuthenticateUserResponse{
+ Authenticated: true,
+ }, nil
+ }
+
+ // Authenticate user
+ // Get IBD creds
+ username, password, err := u.user.GetIBDCreds(ctx, req.Subject)
+ if errors.Is(err, database.ErrIBDCredsNotFound) {
+ return nil, status.New(codes.NotFound, "User has no IDB creds").Err()
+ }
+ if err != nil {
+ return nil, status.Errorf(codes.Internal, "unable to get IBD creds: %v", err)
+ }
+
+ // Authenticate user
+ cookie, err := u.client.Authenticate(ctx, username, password)
+ if errors.Is(err, ibd.ErrBadCredentials) {
+ return &pb.AuthenticateUserResponse{
+ Authenticated: false,
+ }, nil
+ }
+ if err != nil {
+ return nil, status.Errorf(codes.Internal, "unable to authenticate user: %v", err)
+ }
+
+ return &pb.AuthenticateUserResponse{
+ Authenticated: cookie != nil,
+ }, nil
+}
diff --git a/backend/internal/server/server.go b/backend/internal/server/server.go
index c46a629..186d581 100644
--- a/backend/internal/server/server.go
+++ b/backend/internal/server/server.go
@@ -45,7 +45,7 @@ func New(
}
s := grpc.NewServer()
- upb.RegisterUserServiceServer(s, user.New(db, client))
+ upb.RegisterUserServiceServer(s, user.New(db, db, client))
spb.RegisterStockServiceServer(s, stock.New(db, scrapeQueue))
longrunningpb.RegisterOperationsServer(s, newOperationServer(scrapeQueue))
reflection.Register(s)
generated by cgit v1.2.3 (git 2.46.0) at 2025-10-04 20:19:46 +0000