diff options
| author |  Anshul Gupta <ansg191@anshulg.com>
| 2024-08-07 18:56:01 -0700 |
|---|---|---|
| committer |  GitHub <noreply@github.com>
| 2024-08-07 18:56:01 -0700 |
| commit | 08993e2f8497341079010d3d06361c99492c4c07 (patch) | |
| tree | c65d6d571c928410faace1fa51c2ea3f49fce003 | |
| parent | 3de4ebb7560851ccbefe296c197456fe80c22901 (diff) | |
| parent | b8aef1a7fb24815c7d93bc30c7b289b4f5896779 (diff) | |
| download | ibd-trader-08993e2f8497341079010d3d06361c99492c4c07.tar.gz ibd-trader-08993e2f8497341079010d3d06361c99492c4c07.tar.zst ibd-trader-08993e2f8497341079010d3d06361c99492c4c07.zip | |
Merge pull request #1 from ansg191/refactor-database
50 files changed, 650 insertions, 598 deletions
diff --git a/backend/.github/workflows/go.yaml b/backend/.github/workflows/go.yaml index fe1b3a1..2eb8dd2 100644 --- a/backend/.github/workflows/go.yaml +++ b/backend/.github/workflows/go.yaml @@ -14,15 +14,10 @@ jobs: runs-on: ubuntu-latest services: - redis: - image: redis + dind: + image: docker:dind-rootless ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 + - 2375:2375 steps: - uses: actions/checkout@v4 @@ -43,8 +38,6 @@ jobs: run: go build -v ./... - name: Test run: go test -v ./... - env: - REDIS_ADDR: "localhost:6379" lint: runs-on: ubuntu-latest diff --git a/backend/.idea/sqldialects.xml b/backend/.idea/sqldialects.xml index de8a9a3..6df4889 100644 --- a/backend/.idea/sqldialects.xml +++ b/backend/.idea/sqldialects.xml @@ -1,12 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <project version="4"> <component name="SqlDialectMappings"> - <file url="file://$PROJECT_DIR$/db/queries/cookies/get_any_cookie.sql" dialect="PostgreSQL" /> - <file url="file://$PROJECT_DIR$/db/queries/cookies/get_cookies.sql" dialect="PostgreSQL" /> - <file url="file://$PROJECT_DIR$/db/queries/cookies/set_cookie_degraded.sql" dialect="PostgreSQL" /> - <file url="file://$PROJECT_DIR$/db/queries/stocks/add_analysis.sql" dialect="PostgreSQL" /> - <file url="file://$PROJECT_DIR$/db/queries/stocks/add_raw_chart_analysis.sql" dialect="PostgreSQL" /> - <file url="file://$PROJECT_DIR$/db/queries/users/get_ibd_creds.sql" dialect="PostgreSQL" /> <file url="PROJECT" dialect="PostgreSQL" /> </component> </project>
\ No newline at end of file diff --git a/backend/cmd/main.go b/backend/cmd/main.go index 6b49980..2a34780 100644 --- a/backend/cmd/main.go +++ b/backend/cmd/main.go @@ -54,8 +54,14 @@ func main() { "logger.level", level, ) + // Create kms + kms, err := createKMS() + if err != nil { + log.Fatal("Unable to create KMS: ", err) + } + // Connect to the database - db, err := connectDB(logger, cfg) + db, err := connectDB(logger, cfg, kms) defer func(db database.Database) { _ = db.Close() }(db) @@ -78,7 +84,7 @@ func main() { _ = auth // Setup IBD client - client, err := setupIBDClient(cfg, db) + client, err := setupIBDClient(cfg, db, kms) if err != nil { log.Fatal("Unable to setup IBD client: ", err) } @@ -102,7 +108,7 @@ func main() { // Start the gRPC server go func() { - s, err := server.New(ctx, cfg.Server.Port, db, redisClient, client) + s, err := server.New(ctx, cfg.Server.Port, db, redisClient, client, kms, cfg.KMS.GCP.String()) if err != nil { log.Fatal("Unable to create gRPC server: ", err) } @@ -120,6 +126,7 @@ func main() { client, redisClient, db, + kms, analyzer, ) if err != nil { @@ -146,7 +153,7 @@ func main() { ) } -func setupIBDClient(cfg *config.Config, db database.Database) (*ibd.Client, error) { +func setupIBDClient(cfg *config.Config, db database.Database, kms keys.KeyManagementService) (*ibd.Client, error) { pUrl, err := url.Parse(cfg.IBD.ProxyURL) if err != nil { return nil, fmt.Errorf("unable to parse proxy URL: %w", err) @@ -157,18 +164,19 @@ func setupIBDClient(cfg *config.Config, db database.Database) (*ibd.Client, erro transport.NewStandardTransport(&http.Client{Transport: t}), // Default proxied transport scrapfly.New(http.DefaultClient, cfg.IBD.APIKey), // Scrapfly transport } - client := ibd.NewClient(db, transports...) + client := ibd.NewClient(db, kms, transports...) return client, nil } -func connectDB(logger *slog.Logger, cfg *config.Config) (database.Database, error) { +func createKMS() (keys.KeyManagementService, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() + return keys.NewGoogleKMS(ctx) +} - kms, err := keys.NewGoogleKMS(ctx) - if err != nil { - return nil, fmt.Errorf("unable to create Google KMS Client: %w", err) - } +func connectDB(logger *slog.Logger, cfg *config.Config, kms keys.KeyManagementService) (database.Database, error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() db, err := database.New(ctx, logger, cfg.DB.URL, kms, cfg.KMS.GCP.String()) if err != nil { diff --git a/backend/db/embed.go b/backend/db/embed.go index 4302300..e0a4e0b 100644 --- a/backend/db/embed.go +++ b/backend/db/embed.go @@ -4,14 +4,3 @@ import "embed" //go:embed migrations/*.sql var Migrations embed.FS - -//go:embed all:queries -var queries embed.FS - -func GetQuery(name string) (string, error) { - query, err := queries.ReadFile("queries/" + name + ".sql") - if err != nil { - return "", err - } - return string(query), nil -} diff --git a/backend/db/queries/cookies/add_cookie.sql b/backend/db/queries/cookies/add_cookie.sql deleted file mode 100644 index 1519da4..0000000 --- a/backend/db/queries/cookies/add_cookie.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO ibd_tokens (token, expires_at, user_subject, encryption_key) -VALUES ($1, $2, $3, $4)
\ No newline at end of file diff --git a/backend/db/queries/cookies/get_any_cookie.sql b/backend/db/queries/cookies/get_any_cookie.sql deleted file mode 100644 index 4e5c823..0000000 --- a/backend/db/queries/cookies/get_any_cookie.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT ibd_tokens.id, token, encrypted_key, kms_key_name, expires_at -FROM ibd_tokens - INNER JOIN keys ON encryption_key = keys.id -WHERE expires_at > NOW() - AND degraded = FALSE -ORDER BY random() -LIMIT 1; diff --git a/backend/db/queries/cookies/get_cookies.sql b/backend/db/queries/cookies/get_cookies.sql deleted file mode 100644 index 3828ec3..0000000 --- a/backend/db/queries/cookies/get_cookies.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT ibd_tokens.id, token, encrypted_key, kms_key_name, expires_at -FROM ibd_tokens - INNER JOIN keys ON encryption_key = keys.id -WHERE user_subject = $1 - AND expires_at > NOW() - AND degraded = $2 -ORDER BY expires_at DESC;
\ No newline at end of file diff --git a/backend/db/queries/cookies/set_cookie_degraded.sql b/backend/db/queries/cookies/set_cookie_degraded.sql deleted file mode 100644 index 4fd8222..0000000 --- a/backend/db/queries/cookies/set_cookie_degraded.sql +++ /dev/null @@ -1,3 +0,0 @@ -UPDATE ibd_tokens -SET degraded = $1 -WHERE id = $2;
\ No newline at end of file diff --git a/backend/db/queries/keys/add_key.sql b/backend/db/queries/keys/add_key.sql deleted file mode 100644 index bb416c5..0000000 --- a/backend/db/queries/keys/add_key.sql +++ /dev/null @@ -1,3 +0,0 @@ -INSERT INTO keys (kms_key_name, encrypted_key) -VALUES ($1, $2) -RETURNING id;
\ No newline at end of file diff --git a/backend/db/queries/keys/get_key.sql b/backend/db/queries/keys/get_key.sql deleted file mode 100644 index 97d8367..0000000 --- a/backend/db/queries/keys/get_key.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT id, kms_key_name, encrypted_key, created_at -FROM keys -WHERE id = $1;
\ No newline at end of file diff --git a/backend/db/queries/sessions/check_state.sql b/backend/db/queries/sessions/check_state.sql deleted file mode 100644 index dac73e2..0000000 --- a/backend/db/queries/sessions/check_state.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT 1 -FROM sessions -where token = $1;
\ No newline at end of file diff --git a/backend/db/queries/sessions/cleanup_sessions.sql b/backend/db/queries/sessions/cleanup_sessions.sql deleted file mode 100644 index 5f2d22b..0000000 --- a/backend/db/queries/sessions/cleanup_sessions.sql +++ /dev/null @@ -1,2 +0,0 @@ -DELETE FROM sessions -WHERE expires_at < NOW();
\ No newline at end of file diff --git a/backend/db/queries/sessions/create_session.sql b/backend/db/queries/sessions/create_session.sql deleted file mode 100644 index 44f8c56..0000000 --- a/backend/db/queries/sessions/create_session.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO sessions (token, user_subject, access_token, expires_at) -VALUES ($1, $2, $3, $4);
\ No newline at end of file diff --git a/backend/db/queries/sessions/create_state.sql b/backend/db/queries/sessions/create_state.sql deleted file mode 100644 index 577ad7e..0000000 --- a/backend/db/queries/sessions/create_state.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO sessions (token, expires_at) -VALUES ($1, CURRENT_TIMESTAMP + INTERVAL '1 hour');
\ No newline at end of file diff --git a/backend/db/queries/sessions/get_session.sql b/backend/db/queries/sessions/get_session.sql deleted file mode 100644 index 7da8bd0..0000000 --- a/backend/db/queries/sessions/get_session.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT token, user_subject, access_token, expires_at -FROM sessions -WHERE token = $1;
\ No newline at end of file diff --git a/backend/db/queries/stocks/add_analysis.sql b/backend/db/queries/stocks/add_analysis.sql deleted file mode 100644 index 4bb4903..0000000 --- a/backend/db/queries/stocks/add_analysis.sql +++ /dev/null @@ -1,9 +0,0 @@ -UPDATE chart_analysis ca -SET processed = true, - action = $2, - price = $3, - reason = $4, - confidence = $5 -FROM ratings r -WHERE r.id = $1 - AND r.chart_analysis = ca.id
\ No newline at end of file diff --git a/backend/db/queries/stocks/add_rank.sql b/backend/db/queries/stocks/add_rank.sql deleted file mode 100644 index 07f711e..0000000 --- a/backend/db/queries/stocks/add_rank.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO stock_rank (symbol, rank_type, rank) -VALUES ($1, $2, $3);
\ No newline at end of file diff --git a/backend/db/queries/stocks/add_rating.sql b/backend/db/queries/stocks/add_rating.sql deleted file mode 100644 index 6c4baa0..0000000 --- a/backend/db/queries/stocks/add_rating.sql +++ /dev/null @@ -1,3 +0,0 @@ -INSERT INTO ratings (symbol, composite, eps, rel_str, group_rel_str, smr, acc_dis, chart_analysis, price) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) -RETURNING id;
\ No newline at end of file diff --git a/backend/db/queries/stocks/add_raw_chart_analysis.sql b/backend/db/queries/stocks/add_raw_chart_analysis.sql deleted file mode 100644 index a599d87..0000000 --- a/backend/db/queries/stocks/add_raw_chart_analysis.sql +++ /dev/null @@ -1,3 +0,0 @@ -INSERT INTO chart_analysis (raw_analysis) -VALUES ($1) -RETURNING id;
\ No newline at end of file diff --git a/backend/db/queries/stocks/add_stock.sql b/backend/db/queries/stocks/add_stock.sql deleted file mode 100644 index 180a9c3..0000000 --- a/backend/db/queries/stocks/add_stock.sql +++ /dev/null @@ -1,5 +0,0 @@ -INSERT INTO stocks (symbol, name, ibd_url) -VALUES ($1, $2, $3) -ON CONFLICT (symbol) - DO UPDATE SET name = $2, - ibd_url = $3;
\ No newline at end of file diff --git a/backend/db/queries/stocks/get_stock.sql b/backend/db/queries/stocks/get_stock.sql deleted file mode 100644 index cecbd84..0000000 --- a/backend/db/queries/stocks/get_stock.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT symbol, name, ibd_url -FROM stocks -WHERE symbol = $1;
\ No newline at end of file diff --git a/backend/db/queries/stocks/get_stock_info.sql b/backend/db/queries/stocks/get_stock_info.sql deleted file mode 100644 index d4f1bf3..0000000 --- a/backend/db/queries/stocks/get_stock_info.sql +++ /dev/null @@ -1,14 +0,0 @@ -SELECT r.symbol, - s.name, - ca.raw_analysis, - r.composite, - r.eps, - r.rel_str, - r.group_rel_str, - r.smr, - r.acc_dis, - r.price -FROM ratings r - INNER JOIN stocks s on r.symbol = s.symbol - INNER JOIN chart_analysis ca on r.chart_analysis = ca.id -WHERE r.id = $1;
\ No newline at end of file diff --git a/backend/db/queries/users/add_ibd_creds.sql b/backend/db/queries/users/add_ibd_creds.sql deleted file mode 100644 index 054f328..0000000 --- a/backend/db/queries/users/add_ibd_creds.sql +++ /dev/null @@ -1,5 +0,0 @@ -UPDATE users -SET ibd_username = $2, - ibd_password = $3, - encryption_key = $4 -WHERE subject = $1;
\ No newline at end of file diff --git a/backend/db/queries/users/add_user.sql b/backend/db/queries/users/add_user.sql deleted file mode 100644 index bf97ad5..0000000 --- a/backend/db/queries/users/add_user.sql +++ /dev/null @@ -1,3 +0,0 @@ -INSERT INTO users (subject) -VALUES ($1) -ON CONFLICT DO NOTHING;
\ No newline at end of file diff --git a/backend/db/queries/users/get_ibd_creds.sql b/backend/db/queries/users/get_ibd_creds.sql deleted file mode 100644 index 271abcc..0000000 --- a/backend/db/queries/users/get_ibd_creds.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT ibd_username, ibd_password, encrypted_key, kms_key_name -FROM users -INNER JOIN public.keys k on k.id = users.encryption_key -WHERE subject = $1;
\ No newline at end of file diff --git a/backend/db/queries/users/get_user.sql b/backend/db/queries/users/get_user.sql deleted file mode 100644 index 567f988..0000000 --- a/backend/db/queries/users/get_user.sql +++ /dev/null @@ -1,3 +0,0 @@ -SELECT subject, ibd_username, ibd_password, encryption_key -FROM users -WHERE subject = $1;
\ No newline at end of file diff --git a/backend/db/queries/users/list_users.sql b/backend/db/queries/users/list_users.sql deleted file mode 100644 index ceafeb2..0000000 --- a/backend/db/queries/users/list_users.sql +++ /dev/null @@ -1,2 +0,0 @@ -SELECT subject, ibd_username, ibd_password, encryption_key -FROM users;
\ No newline at end of file diff --git a/backend/go.mod b/backend/go.mod index cdd1e1c..7b717b2 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -18,6 +18,7 @@ require ( github.com/lib/pq v1.10.9 github.com/lmittmann/tint v1.0.5 github.com/mennanov/fmutils v0.3.0 + github.com/ory/dockertest/v3 v3.10.0 github.com/redis/go-redis/v9 v9.6.1 github.com/robfig/cron/v3 v3.0.1 github.com/sashabaranov/go-openai v1.27.1 @@ -38,34 +39,55 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/iam v1.1.12 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/PuerkitoBio/goquery v1.9.2 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/containerd/continuity v0.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/docker/cli v20.10.17+incompatible // indirect + github.com/docker/docker v24.0.9+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/s2a-go v0.1.8 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/opencontainers/runc v1.1.5 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sirupsen/logrus v1.9.2 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect @@ -76,13 +98,16 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/mod v0.19.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.23.0 // indirect google.golang.org/api v0.190.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240730163845-b1a4ccb954bf // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.3.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index d7235e3..a5a8ea5 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -399,6 +399,8 @@ github.com/EDDYCJY/fake-useragent v0.2.0 h1:Jcnkk2bgXmDpX0z+ELlUErTkoLb/mxFBNd2Y github.com/EDDYCJY/fake-useragent v0.2.0/go.mod h1:5wn3zzlDxhKW6NYknushqinPcAqZcAPHy8lLczCdJdc= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= @@ -415,15 +417,19 @@ github.com/bsm/redislock v0.9.4 h1:X/Wse1DPpiQgHbVYRE9zv6m070UcKoOGekgvpNhiSvw= github.com/bsm/redislock v0.9.4/go.mod h1:Epf7AJLiSFwLCiZcfi6pWFO/8eAYrYpQXFxEDPoDeAk= github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU= github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -434,8 +440,16 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= +github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI= github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -444,12 +458,15 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dhui/dktest v0.4.1 h1:/w+IWuDXVymg3IrRJCHHOkMK10m9aNVMOyD0X12YVTg= github.com/dhui/dktest v0.4.1/go.mod h1:DdOqcUpL7vgyP4GlF3X3w7HbSlz8cEQzwewPveYEQbA= +github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M= +github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -464,6 +481,7 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -479,6 +497,10 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-migrate/migrate/v4 v4.17.1 h1:4zQ6iqL6t6AiItphxJctQb3cFqWiSpMnX7wLTPnnYO4= @@ -558,6 +580,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= @@ -590,10 +614,13 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -615,14 +642,22 @@ github.com/mennanov/fmutils v0.3.0 h1:2YSyrO8oOLQQwB/iKe+xDDGO6xCUHiIAj3gYhY7D4A github.com/mennanov/fmutils v0.3.0/go.mod h1:ph1jsu8gV1gUgMURCmfIVbXKG3O2/O5o/UbPbbqu8zs= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= +github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= +github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -639,12 +674,18 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sashabaranov/go-openai v1.27.1 h1:7Nx6db5NXbcoutNmAUQulEQZEpHG/SkzfexP2X5RWMk= github.com/sashabaranov/go-openai v1.27.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= +github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -660,6 +701,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -672,6 +715,16 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -785,6 +838,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= @@ -857,9 +911,12 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -881,6 +938,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -895,7 +953,10 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -910,6 +971,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -979,6 +1041,7 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -987,6 +1050,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1244,9 +1308,13 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo= +gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/backend/internal/database/cookies.go b/backend/internal/database/cookies.go index 8bed854..d652b65 100644 --- a/backend/internal/database/cookies.go +++ b/backend/internal/database/cookies.go @@ -11,29 +11,21 @@ import ( "github.com/ansg191/ibd-trader-backend/internal/keys" ) -type CookieStore interface { - CookieSource - AddCookie(ctx context.Context, subject string, cookie *http.Cookie) error - RepairCookie(ctx context.Context, id uint) error -} - -type CookieSource interface { - GetAnyCookie(ctx context.Context) (*IBDCookie, error) - GetCookies(ctx context.Context, subject string, degraded bool) ([]IBDCookie, error) - ReportCookieFailure(ctx context.Context, id uint) error -} - -func (d *database) GetAnyCookie(ctx context.Context) (*IBDCookie, error) { - row, err := d.queryRow(ctx, d.db, "cookies/get_any_cookie") - if err != nil { - return nil, fmt.Errorf("unable to get any ibd cookie: %w", err) - } +func GetAnyCookie(ctx context.Context, exec Executor, kms keys.KeyManagementService) (*IBDCookie, error) { + row := exec.QueryRowContext(ctx, ` +SELECT ibd_tokens.id, token, encrypted_key, kms_key_name, expires_at +FROM ibd_tokens + INNER JOIN keys ON encryption_key = keys.id +WHERE expires_at > NOW() + AND degraded = FALSE +ORDER BY random() +LIMIT 1;`) var id uint var encryptedToken, encryptedKey []byte var keyName string var expiry time.Time - err = row.Scan(&id, &encryptedToken, &encryptedKey, &keyName, &expiry) + err := row.Scan(&id, &encryptedToken, &encryptedKey, &keyName, &expiry) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil @@ -41,7 +33,11 @@ func (d *database) GetAnyCookie(ctx context.Context) (*IBDCookie, error) { return nil, fmt.Errorf("unable to scan sql row into ibd cookie: %w", err) } - token, err := keys.Decrypt(ctx, d.kms, keyName, encryptedToken, encryptedKey) + // Set the expiry to UTC explicitly. + // For some reason, the expiry time is set to location="". + expiry = expiry.UTC() + + token, err := keys.Decrypt(ctx, kms, keyName, encryptedToken, encryptedKey) if err != nil { return nil, fmt.Errorf("unable to decrypt token: %w", err) } @@ -51,24 +47,41 @@ func (d *database) GetAnyCookie(ctx context.Context) (*IBDCookie, error) { }, nil } -func (d *database) GetCookies(ctx context.Context, subject string, degraded bool) ([]IBDCookie, error) { - row, err := d.query(ctx, d.db, "cookies/get_cookies", subject, degraded) +func GetCookies( + ctx context.Context, + exec Executor, + kms keys.KeyManagementService, + subject string, + degraded bool, +) ([]IBDCookie, error) { + rows, err := exec.QueryContext(ctx, ` +SELECT ibd_tokens.id, token, encrypted_key, kms_key_name, expires_at +FROM ibd_tokens + INNER JOIN keys ON encryption_key = keys.id +WHERE user_subject = $1 + AND expires_at > NOW() + AND degraded = $2 +ORDER BY expires_at DESC;`, subject, degraded) if err != nil { return nil, fmt.Errorf("unable to get ibd cookies: %w", err) } cookies := make([]IBDCookie, 0) - for row.Next() { + for rows.Next() { var id uint var encryptedToken, encryptedKey []byte var keyName string var expiry time.Time - err = row.Scan(&id, &encryptedToken, &encryptedKey, &keyName, &expiry) + err = rows.Scan(&id, &encryptedToken, &encryptedKey, &keyName, &expiry) if err != nil { return nil, fmt.Errorf("unable to scan sql row into ibd cookie: %w", err) } - token, err := keys.Decrypt(ctx, d.kms, keyName, encryptedToken, encryptedKey) + // Set the expiry to UTC explicitly. + // For some reason, the expiry time is set to location="". + expiry = expiry.UTC() + + token, err := keys.Decrypt(ctx, kms, keyName, encryptedToken, encryptedKey) if err != nil { return nil, fmt.Errorf("unable to decrypt token: %w", err) } @@ -83,9 +96,15 @@ func (d *database) GetCookies(ctx context.Context, subject string, degraded bool return cookies, nil } -func (d *database) AddCookie(ctx context.Context, subject string, cookie *http.Cookie) error { +func AddCookie( + ctx context.Context, + exec Executor, + kms keys.KeyManagementService, + subject string, + cookie *http.Cookie, +) error { // Get the key ID for the user - user, err := d.GetUser(ctx, subject) + user, err := GetUser(ctx, exec, subject) if err != nil { return fmt.Errorf("unable to get user: %w", err) } @@ -94,19 +113,21 @@ func (d *database) AddCookie(ctx context.Context, subject string, cookie *http.C } // Get the key - key, err := d.GetKey(ctx, *user.EncryptionKeyID) + key, err := GetKey(ctx, exec, *user.EncryptionKeyID) if err != nil { return fmt.Errorf("unable to get key: %w", err) } // Encrypt the token - encryptedToken, err := keys.EncryptWithKey(ctx, d.kms, key.Name, key.Key, []byte(cookie.Value)) + encryptedToken, err := keys.EncryptWithKey(ctx, kms, key.Name, key.Key, []byte(cookie.Value)) if err != nil { return fmt.Errorf("unable to encrypt token: %w", err) } // Add the cookie to the database - _, err = d.exec(ctx, d.db, "cookies/add_cookie", encryptedToken, cookie.Expires, subject, key.Id) + _, err = exec.ExecContext(ctx, ` +INSERT INTO ibd_tokens (token, expires_at, user_subject, encryption_key) +VALUES ($1, $2, $3, $4)`, encryptedToken, cookie.Expires, subject, key.Id) if err != nil { return fmt.Errorf("unable to add cookie: %w", err) } @@ -114,16 +135,22 @@ func (d *database) AddCookie(ctx context.Context, subject string, cookie *http.C return nil } -func (d *database) ReportCookieFailure(ctx context.Context, id uint) error { - _, err := d.exec(ctx, d.db, "cookies/set_cookie_degraded", true, id) +func ReportCookieFailure(ctx context.Context, exec Executor, id uint) error { + _, err := exec.ExecContext(ctx, ` +UPDATE ibd_tokens +SET degraded = TRUE +WHERE id = $1;`, id) if err != nil { return fmt.Errorf("unable to report cookie failure: %w", err) } return nil } -func (d *database) RepairCookie(ctx context.Context, id uint) error { - _, err := d.exec(ctx, d.db, "cookies/set_cookie_degraded", false, id) +func RepairCookie(ctx context.Context, exec Executor, id uint) error { + _, err := exec.ExecContext(ctx, ` +UPDATE ibd_tokens +SET degraded = FALSE +WHERE id = $1;`, id) if err != nil { return fmt.Errorf("unable to report cookie failure: %w", err) } diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go index 3c822bc..409dd3c 100644 --- a/backend/internal/database/database.go +++ b/backend/internal/database/database.go @@ -5,7 +5,6 @@ import ( "database/sql" "database/sql/driver" "errors" - "fmt" "io" "log/slog" "sync" @@ -22,12 +21,7 @@ import ( type Database interface { io.Closer - UserStore - CookieStore - KeyStore - SessionStore - StockStore - + TransactionExecutor driver.Pinger Migrate(ctx context.Context) error @@ -70,24 +64,7 @@ func (d *database) Close() error { } func (d *database) Migrate(ctx context.Context) error { - fs, err := iofs.New(db.Migrations, "migrations") - if err != nil { - return err - } - - m, err := migrate.NewWithSourceInstance("iofs", fs, d.url) - if err != nil { - return err - } - - d.logger.InfoContext(ctx, "Running DB migration") - err = m.Up() - if err != nil && !errors.Is(err, migrate.ErrNoChange) { - d.logger.ErrorContext(ctx, "DB migration failed", "error", err) - return err - } - - return nil + return Migrate(ctx, d.url) } func (d *database) Maintenance(ctx context.Context) { @@ -101,11 +78,9 @@ func (d *database) Maintenance(ctx context.Context) { var wg sync.WaitGroup wg.Add(1) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + _, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() - go d.cleanupSessions(ctx, &wg) - wg.Wait() }() case <-ctx.Done(): @@ -114,65 +89,78 @@ func (d *database) Maintenance(ctx context.Context) { } } -func (d *database) Ping(ctx context.Context) error { - return d.db.PingContext(ctx) -} - -func (d *database) execInternal(ctx context.Context, queryName string, fn func(string) (any, error)) (any, error) { - query, err := db.GetQuery(queryName) +func Migrate(ctx context.Context, url string) error { + fs, err := iofs.New(db.Migrations, "migrations") if err != nil { - return nil, fmt.Errorf("unable to get query: %w", err) + return err } - d.logger.DebugContext(ctx, "Executing query", "name", queryName, "query", query) - - now := time.Now() - // Execute the query - result, err := fn(query) + m, err := migrate.NewWithSourceInstance("iofs", fs, url) if err != nil { - return nil, fmt.Errorf("unable to execute query: %w", err) + return err } - d.logger.DebugContext(ctx, "Query executed successfully", "name", queryName, "duration", time.Since(now)) + slog.InfoContext(ctx, "Running DB migration") + err = m.Up() + if err != nil && !errors.Is(err, migrate.ErrNoChange) { + slog.ErrorContext(ctx, "DB migration failed", "error", err) + return err + } - return result, nil + return nil } -func (d *database) exec(ctx context.Context, exec executor, queryName string, args ...any) (sql.Result, error) { - ret, err := d.execInternal(ctx, queryName, func(query string) (any, error) { - return exec.ExecContext(ctx, query, args...) - }) - if err != nil { - return nil, err - } else { - return ret.(sql.Result), nil - } +func (d *database) Ping(ctx context.Context) error { + return d.db.PingContext(ctx) +} + +type Executor interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +type TransactionExecutor interface { + Executor + BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) } -func (d *database) query(ctx context.Context, exec executor, queryName string, args ...any) (*sql.Rows, error) { - ret, err := d.execInternal(ctx, queryName, func(query string) (any, error) { - return exec.QueryContext(ctx, query, args...) - }) +func (d *database) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + d.logger.DebugContext(ctx, "Executing query", "query", query) + + now := time.Now() + ret, err := d.db.ExecContext(ctx, query, args...) if err != nil { return nil, err - } else { - return ret.(*sql.Rows), nil } + + d.logger.DebugContext(ctx, "Query executed successfully", "duration", time.Since(now)) + return ret, nil } -func (d *database) queryRow(ctx context.Context, exec executor, queryName string, args ...any) (*sql.Row, error) { - ret, err := d.execInternal(ctx, queryName, func(query string) (any, error) { - return exec.QueryRowContext(ctx, query, args...), nil - }) +func (d *database) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + d.logger.DebugContext(ctx, "Executing query", "query", query) + + now := time.Now() + ret, err := d.db.QueryContext(ctx, query, args...) if err != nil { return nil, err - } else { - return ret.(*sql.Row), nil } + + d.logger.DebugContext(ctx, "Query executed successfully", "duration", time.Since(now)) + return ret, nil } -type executor interface { - ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) - QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) - QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +func (d *database) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row { + d.logger.DebugContext(ctx, "Executing query", "query", query) + + now := time.Now() + ret := d.db.QueryRowContext(ctx, query, args...) + + d.logger.DebugContext(ctx, "Query executed successfully", "duration", time.Since(now)) + return ret +} + +func (d *database) BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) { + return d.db.BeginTx(ctx, opts) } diff --git a/backend/internal/database/keys.go b/backend/internal/database/keys.go index 0ec4b67..e2e2770 100644 --- a/backend/internal/database/keys.go +++ b/backend/internal/database/keys.go @@ -6,19 +6,14 @@ import ( "time" ) -type KeyStore interface { - AddKey(ctx context.Context, keyName string, key []byte) (int, error) - GetKey(ctx context.Context, keyId int) (*Key, error) -} - -func (d *database) AddKey(ctx context.Context, keyName string, key []byte) (int, error) { - row, err := d.queryRow(ctx, d.db, "keys/add_key", keyName, key) - if err != nil { - return 0, fmt.Errorf("unable to add key: %w", err) - } +func AddKey(ctx context.Context, exec Executor, keyName string, key []byte) (int, error) { + row := exec.QueryRowContext(ctx, ` +INSERT INTO keys (kms_key_name, encrypted_key) +VALUES ($1, $2) +RETURNING id;`, keyName, key) var keyId int - err = row.Scan(&keyId) + err := row.Scan(&keyId) if err != nil { return 0, fmt.Errorf("unable to scan key id: %w", err) } @@ -26,14 +21,14 @@ func (d *database) AddKey(ctx context.Context, keyName string, key []byte) (int, return keyId, nil } -func (d *database) GetKey(ctx context.Context, keyId int) (*Key, error) { - row, err := d.queryRow(ctx, d.db, "keys/get_key", keyId) - if err != nil { - return nil, fmt.Errorf("unable to get key: %w", err) - } +func GetKey(ctx context.Context, exec Executor, keyId int) (*Key, error) { + row := exec.QueryRowContext(ctx, ` +SELECT id, kms_key_name, encrypted_key, created_at +FROM keys +WHERE id = $1;`, keyId) key := &Key{} - err = row.Scan(&key.Id, &key.Name, &key.Key, &key.Created) + err := row.Scan(&key.Id, &key.Name, &key.Key, &key.Created) if err != nil { return nil, fmt.Errorf("unable to scan key: %w", err) } diff --git a/backend/internal/database/session.go b/backend/internal/database/session.go deleted file mode 100644 index 36867b3..0000000 --- a/backend/internal/database/session.go +++ /dev/null @@ -1,122 +0,0 @@ -package database - -import ( - "context" - "crypto/rand" - "database/sql" - "encoding/base64" - "errors" - "io" - "sync" - - "github.com/coreos/go-oidc/v3/oidc" - "golang.org/x/oauth2" -) - -type SessionStore interface { - CreateState(ctx context.Context) (string, error) - CheckState(ctx context.Context, state string) (bool, error) - CreateSession(ctx context.Context, token *oauth2.Token, idToken *oidc.IDToken) (string, error) - GetSession(ctx context.Context, sessionToken string) (*Session, error) -} - -func (d *database) CreateState(ctx context.Context) (string, error) { - // Generate a random CSRF state token - tokenBytes := make([]byte, 32) - if _, err := io.ReadFull(rand.Reader, tokenBytes); err != nil { - return "", err - } - token := base64.URLEncoding.EncodeToString(tokenBytes) - - // Insert the state into the database - _, err := d.exec(ctx, d.db, "sessions/create_state", token) - if err != nil { - return "", err - } - - return token, nil -} - -func (d *database) CheckState(ctx context.Context, state string) (bool, error) { - var exists bool - row, err := d.queryRow(ctx, d.db, "sessions/check_state", state) - if err != nil { - return false, err - } - err = row.Scan(&exists) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return false, nil - } - return false, err - } - return exists, nil -} - -func (d *database) CreateSession( - ctx context.Context, - token *oauth2.Token, - idToken *oidc.IDToken, -) (sessionToken string, err error) { - // Generate a random session token - tokenBytes := make([]byte, 32) - if _, err = io.ReadFull(rand.Reader, tokenBytes); err != nil { - return - } - sessionToken = base64.URLEncoding.EncodeToString(tokenBytes) - - // Insert the session into the database - _, err = d.exec( - ctx, - d.db, - "sessions/create_session", - sessionToken, - idToken.Subject, - token.AccessToken, - token.Expiry, - ) - return -} - -func (d *database) GetSession(ctx context.Context, sessionToken string) (*Session, error) { - row, err := d.queryRow(ctx, d.db, "sessions/get_session", sessionToken) - if err != nil { - d.logger.ErrorContext(ctx, "Failed to get session", "error", err) - return nil, err - } - - var session Session - err = row.Scan(&session.Token, &session.Subject, &session.OAuthToken.AccessToken, &session.OAuthToken.Expiry) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - d.logger.ErrorContext(ctx, "Failed to scan session", "error", err) - return nil, err - } - - return &session, nil -} - -func (d *database) cleanupSessions(ctx context.Context, wg *sync.WaitGroup) { - defer wg.Done() - - result, err := d.exec(ctx, d.db, "sessions/cleanup_sessions") - if err != nil { - d.logger.Error("Failed to clean up sessions", "error", err) - return - } - - rows, err := result.RowsAffected() - if err != nil { - d.logger.ErrorContext(ctx, "Failed to get rows affected", "error", err) - return - } - d.logger.DebugContext(ctx, "Cleaned up sessions", "rows", rows) -} - -type Session struct { - Token string - Subject string - OAuthToken oauth2.Token -} diff --git a/backend/internal/database/stocks.go b/backend/internal/database/stocks.go index f74e4e8..865aec4 100644 --- a/backend/internal/database/stocks.go +++ b/backend/internal/database/stocks.go @@ -14,23 +14,15 @@ import ( var ErrStockNotFound = errors.New("stock not found") -type StockStore interface { - GetStock(ctx context.Context, symbol string) (Stock, error) - AddStock(ctx context.Context, stock Stock) error - AddRanking(ctx context.Context, symbol string, ibd50, cap20 int) error - AddStockInfo(ctx context.Context, info *StockInfo) (string, error) - GetStockInfo(ctx context.Context, id string) (*StockInfo, error) - AddAnalysis(ctx context.Context, ratingId string, analysis *analyzer.Analysis) error -} - -func (d *database) GetStock(ctx context.Context, symbol string) (Stock, error) { - row, err := d.queryRow(ctx, d.db, "stocks/get_stock", symbol) - if err != nil { - return Stock{}, err - } +func GetStock(ctx context.Context, exec Executor, symbol string) (Stock, error) { + row := exec.QueryRowContext(ctx, ` +SELECT symbol, name, ibd_url +FROM stocks +WHERE symbol = $1; +`, symbol) var stock Stock - if err = row.Scan(&stock.Symbol, &stock.Name, &stock.IBDUrl); err != nil { + if err := row.Scan(&stock.Symbol, &stock.Name, &stock.IBDUrl); err != nil { if errors.Is(err, sql.ErrNoRows) { return Stock{}, ErrStockNotFound } @@ -40,20 +32,29 @@ func (d *database) GetStock(ctx context.Context, symbol string) (Stock, error) { return stock, nil } -func (d *database) AddStock(ctx context.Context, stock Stock) error { - _, err := d.exec(ctx, d.db, "stocks/add_stock", stock.Symbol, stock.Name, stock.IBDUrl) +func AddStock(ctx context.Context, exec Executor, stock Stock) error { + _, err := exec.ExecContext(ctx, ` +INSERT INTO stocks (symbol, name, ibd_url) +VALUES ($1, $2, $3) +ON CONFLICT (symbol) + DO UPDATE SET name = $2, + ibd_url = $3;`, stock.Symbol, stock.Name, stock.IBDUrl) return err } -func (d *database) AddRanking(ctx context.Context, symbol string, ibd50, cap20 int) error { +func AddRanking(ctx context.Context, exec Executor, symbol string, ibd50, cap20 int) error { if ibd50 > 0 { - _, err := d.exec(ctx, d.db, "stocks/add_rank", symbol, "ibd50", ibd50) + _, err := exec.ExecContext(ctx, ` +INSERT INTO stock_rank (symbol, rank_type, rank) +VALUES ($1, $2, $3)`, symbol, "ibd50", ibd50) if err != nil { return err } } if cap20 > 0 { - _, err := d.exec(ctx, d.db, "stocks/add_rank", symbol, "cap20", cap20) + _, err := exec.ExecContext(ctx, ` +INSERT INTO stock_rank (symbol, rank_type, rank) +VALUES ($1, $2, $3)`, symbol, "cap20", cap20) if err != nil { return err } @@ -61,8 +62,8 @@ func (d *database) AddRanking(ctx context.Context, symbol string, ibd50, cap20 i return nil } -func (d *database) AddStockInfo(ctx context.Context, info *StockInfo) (string, error) { - tx, err := d.db.BeginTx(ctx, nil) +func AddStockInfo(ctx context.Context, exec TransactionExecutor, info *StockInfo) (string, error) { + tx, err := exec.BeginTx(ctx, nil) if err != nil { return "", err } @@ -71,10 +72,10 @@ func (d *database) AddStockInfo(ctx context.Context, info *StockInfo) (string, e }(tx) // Add raw chart analysis - row, err := d.queryRow(ctx, tx, "stocks/add_raw_chart_analysis", info.ChartAnalysis) - if err != nil { - return "", err - } + row := tx.QueryRowContext(ctx, ` +INSERT INTO chart_analysis (raw_analysis) +VALUES ($1) +RETURNING id;`, info.ChartAnalysis) var chartAnalysisID string if err = row.Scan(&chartAnalysisID); err != nil { @@ -82,8 +83,11 @@ func (d *database) AddStockInfo(ctx context.Context, info *StockInfo) (string, e } // Add stock info - row, err = d.queryRow(ctx, tx, - "stocks/add_rating", + row = tx.QueryRowContext(ctx, + ` +INSERT INTO ratings (symbol, composite, eps, rel_str, group_rel_str, smr, acc_dis, chart_analysis, price) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +RETURNING id;`, info.Symbol, info.Ratings.Composite, info.Ratings.EPS, @@ -94,9 +98,6 @@ func (d *database) AddStockInfo(ctx context.Context, info *StockInfo) (string, e chartAnalysisID, info.Price.Display(), ) - if err != nil { - return "", err - } var ratingsID string if err = row.Scan(&ratingsID); err != nil { @@ -106,15 +107,26 @@ func (d *database) AddStockInfo(ctx context.Context, info *StockInfo) (string, e return ratingsID, tx.Commit() } -func (d *database) GetStockInfo(ctx context.Context, id string) (*StockInfo, error) { - row, err := d.queryRow(ctx, d.db, "stocks/get_stock_info", id) - if err != nil { - return nil, err - } +func GetStockInfo(ctx context.Context, exec Executor, id string) (*StockInfo, error) { + row := exec.QueryRowContext(ctx, ` +SELECT r.symbol, + s.name, + ca.raw_analysis, + r.composite, + r.eps, + r.rel_str, + r.group_rel_str, + r.smr, + r.acc_dis, + r.price +FROM ratings r + INNER JOIN stocks s on r.symbol = s.symbol + INNER JOIN chart_analysis ca on r.chart_analysis = ca.id +WHERE r.id = $1;`, id) var info StockInfo var priceStr string - err = row.Scan( + err := row.Scan( &info.Symbol, &info.Name, &info.ChartAnalysis, @@ -138,8 +150,17 @@ func (d *database) GetStockInfo(ctx context.Context, id string) (*StockInfo, err return &info, nil } -func (d *database) AddAnalysis(ctx context.Context, ratingId string, analysis *analyzer.Analysis) error { - _, err := d.exec(ctx, d.db, "stocks/add_analysis", +func AddAnalysis(ctx context.Context, exec Executor, ratingId string, analysis *analyzer.Analysis) error { + _, err := exec.ExecContext(ctx, ` +UPDATE chart_analysis ca +SET processed = true, + action = $2, + price = $3, + reason = $4, + confidence = $5 +FROM ratings r +WHERE r.id = $1 + AND r.chart_analysis = ca.id;`, ratingId, analysis.Action, analysis.Price.Display(), diff --git a/backend/internal/database/users.go b/backend/internal/database/users.go index ff6f674..d023598 100644 --- a/backend/internal/database/users.go +++ b/backend/internal/database/users.go @@ -9,35 +9,25 @@ import ( "github.com/ansg191/ibd-trader-backend/internal/keys" ) -type UserStore interface { - AddUser(ctx context.Context, subject string) error - GetUser(ctx context.Context, subject string) (*User, error) - ListUsers(ctx context.Context, hasIBDCreds bool) ([]User, error) - AddIBDCreds(ctx context.Context, subject string, username string, password string) error - GetIBDCreds(ctx context.Context, subject string) (username string, password string, err error) -} - var ErrUserNotFound = fmt.Errorf("user not found") var ErrIBDCredsNotFound = fmt.Errorf("ibd creds not found") -func (d *database) AddUser(ctx context.Context, subject string) (err error) { - _, err = d.exec( - ctx, - d.db, - "users/add_user", - subject, - ) +func AddUser(ctx context.Context, exec Executor, subject string) (err error) { + _, err = exec.ExecContext(ctx, ` +INSERT INTO users (subject) +VALUES ($1) +ON CONFLICT DO NOTHING;`, subject) return } -func (d *database) GetUser(ctx context.Context, subject string) (*User, error) { - row, err := d.queryRow(ctx, d.db, "users/get_user", subject) - if err != nil { - return nil, fmt.Errorf("unable to get user: %w", err) - } +func GetUser(ctx context.Context, exec Executor, subject string) (*User, error) { + row := exec.QueryRowContext(ctx, ` +SELECT subject, ibd_username, ibd_password, encryption_key +FROM users +WHERE subject = $1;`, subject) user := &User{} - err = row.Scan(&user.Subject, &user.IBDUsername, &user.EncryptedIBDPassword, &user.EncryptionKeyID) + err := row.Scan(&user.Subject, &user.IBDUsername, &user.EncryptedIBDPassword, &user.EncryptionKeyID) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrUserNotFound @@ -48,8 +38,11 @@ func (d *database) GetUser(ctx context.Context, subject string) (*User, error) { return user, nil } -func (d *database) ListUsers(ctx context.Context, hasIBDCreds bool) ([]User, error) { - rows, err := d.query(ctx, d.db, "users/list_users") +func ListUsers(ctx context.Context, exec Executor, hasIBDCreds bool) ([]User, error) { + rows, err := exec.QueryContext(ctx, ` +SELECT subject, ibd_username, ibd_password, encryption_key +FROM users; +`) if err != nil { return nil, fmt.Errorf("unable to list users: %w", err) } @@ -71,13 +64,18 @@ func (d *database) ListUsers(ctx context.Context, hasIBDCreds bool) ([]User, err return users, nil } -func (d *database) AddIBDCreds(ctx context.Context, subject string, username string, password string) error { - encryptedPass, encryptedKey, err := keys.Encrypt(ctx, d.kms, d.keyName, []byte(password)) +func AddIBDCreds( + ctx context.Context, + exec TransactionExecutor, + kms keys.KeyManagementService, + keyName, subject, username, password string, +) error { + encryptedPass, encryptedKey, err := keys.Encrypt(ctx, kms, keyName, []byte(password)) if err != nil { return fmt.Errorf("unable to encrypt password: %w", err) } - tx, err := d.db.BeginTx(ctx, nil) + tx, err := exec.BeginTx(ctx, nil) if err != nil { return err } @@ -85,18 +83,17 @@ func (d *database) AddIBDCreds(ctx context.Context, subject string, username str _ = tx.Rollback() }(tx) - row, err := d.queryRow(ctx, tx, "keys/add_key", d.keyName, encryptedKey) + keyId, err := AddKey(ctx, tx, keyName, encryptedKey) if err != nil { return fmt.Errorf("unable to add ibd creds key: %w", err) } - var keyId int - err = row.Scan(&keyId) - if err != nil { - return fmt.Errorf("unable to scan key id: %w", err) - } - - _, err = d.exec(ctx, tx, "users/add_ibd_creds", subject, username, encryptedPass, keyId) + _, err = exec.ExecContext(ctx, ` +UPDATE users +SET ibd_username = $2, + ibd_password = $3, + encryption_key = $4 +WHERE subject = $1;`, subject, username, encryptedPass, keyId) if err != nil { return fmt.Errorf("unable to add ibd creds to user: %w", err) } @@ -108,11 +105,21 @@ func (d *database) AddIBDCreds(ctx context.Context, subject string, username str return nil } -func (d *database) GetIBDCreds(ctx context.Context, subject string) (username string, password string, err error) { - row, err := d.queryRow(ctx, d.db, "users/get_ibd_creds", subject) - if err != nil { - return "", "", fmt.Errorf("unable to get ibd creds: %w", err) - } +func GetIBDCreds( + ctx context.Context, + exec Executor, + kms keys.KeyManagementService, + subject string, +) ( + username string, + password string, + err error, +) { + row := exec.QueryRowContext(ctx, ` +SELECT ibd_username, ibd_password, encrypted_key, kms_key_name +FROM users +INNER JOIN public.keys k on k.id = users.encryption_key +WHERE subject = $1;`, subject) var encryptedPass, encryptedKey []byte var keyName string @@ -124,7 +131,7 @@ func (d *database) GetIBDCreds(ctx context.Context, subject string) (username st return "", "", fmt.Errorf("unable to scan sql row into ibd creds: %w", err) } - passwordBytes, err := keys.Decrypt(ctx, d.kms, keyName, encryptedPass, encryptedKey) + passwordBytes, err := keys.Decrypt(ctx, kms, keyName, encryptedPass, encryptedKey) if err != nil { return "", "", fmt.Errorf("unable to decrypt password: %w", err) } diff --git a/backend/internal/ibd/auth_test.go b/backend/internal/ibd/auth_test.go index 54ea98a..157b507 100644 --- a/backend/internal/ibd/auth_test.go +++ b/backend/internal/ibd/auth_test.go @@ -163,7 +163,7 @@ func TestClient_Authenticate(t *testing.T) { return resp, nil }) - client := NewClient(nil, newTransport(tp)) + client := NewClient(nil, nil, newTransport(tp)) cookie, err := client.Authenticate(context.Background(), "abc", "xyz") require.NoError(t, err) @@ -189,7 +189,7 @@ func TestClient_Authenticate_401(t *testing.T) { return httpmock.NewStringResponse(http.StatusUnauthorized, `{"name":"ValidationError","code":"ERR016","message":"Wrong username or password","description":"Wrong username or password"}`), nil }) - client := NewClient(nil, newTransport(tp)) + client := NewClient(nil, nil, newTransport(tp)) cookie, err := client.Authenticate(context.Background(), "abc", "xyz") assert.Nil(t, cookie) diff --git a/backend/internal/ibd/client.go b/backend/internal/ibd/client.go index 25c5173..c8575e3 100644 --- a/backend/internal/ibd/client.go +++ b/backend/internal/ibd/client.go @@ -9,6 +9,7 @@ import ( "github.com/ansg191/ibd-trader-backend/internal/database" "github.com/ansg191/ibd-trader-backend/internal/ibd/transport" + "github.com/ansg191/ibd-trader-backend/internal/keys" ) var ErrNoAvailableCookies = errors.New("no available cookies") @@ -16,20 +17,22 @@ var ErrNoAvailableTransports = errors.New("no available transports") type Client struct { transports []transport.Transport - cookies database.CookieSource + db database.Executor + kms keys.KeyManagementService } func NewClient( - cookies database.CookieSource, + db database.Executor, + kms keys.KeyManagementService, transports ...transport.Transport, ) *Client { - return &Client{transports, cookies} + return &Client{transports, db, kms} } func (c *Client) getCookie(ctx context.Context, subject *string) (uint, *http.Cookie, error) { if subject == nil { // No subject requirement, get any cookie - cookie, err := c.cookies.GetAnyCookie(ctx) + cookie, err := database.GetAnyCookie(ctx, c.db, c.kms) if err != nil { return 0, nil, err } @@ -41,7 +44,7 @@ func (c *Client) getCookie(ctx context.Context, subject *string) (uint, *http.Co } // Get cookie by subject - cookies, err := c.cookies.GetCookies(ctx, *subject, false) + cookies, err := database.GetCookies(ctx, c.db, c.kms, *subject, false) if err != nil { return 0, nil, err } diff --git a/backend/internal/ibd/client_test.go b/backend/internal/ibd/client_test.go index d2dc1b2..2368a31 100644 --- a/backend/internal/ibd/client_test.go +++ b/backend/internal/ibd/client_test.go @@ -2,30 +2,155 @@ package ibd import ( "context" + "database/sql" + "fmt" + "log" + "math/rand/v2" "testing" "time" "github.com/ansg191/ibd-trader-backend/internal/database" + "github.com/ansg191/ibd-trader-backend/internal/keys" + _ "github.com/lib/pq" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestClient_getCookie(t *testing.T) { - t.Parallel() +var ( + db *sql.DB + maxTime = time.Date(2100, 1, 1, 0, 0, 0, 0, time.UTC) + letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") +) - t.Run("no cookies", func(t *testing.T) { - t.Parallel() +func TestMain(m *testing.M) { + pool, err := dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not create pool: %s", err) + } + + err = pool.Client.Ping() + if err != nil { + log.Fatalf("Could not connect to Docker: %s", err) + } + + resource, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "postgres", + Tag: "16", + Env: []string{ + "POSTGRES_PASSWORD=secret", + "POSTGRES_USER=ibd-client-test", + "POSTGRES_DB=ibd-client-test", + "listen_addresses='*'", + }, + Cmd: []string{ + "postgres", + "-c", + "log_statement=all", + }, + }, func(config *docker.HostConfig) { + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{Name: "no"} + }) + if err != nil { + log.Fatalf("Could not start resource: %s", err) + } + + hostAndPort := resource.GetHostPort("5432/tcp") + databaseUrl := fmt.Sprintf("postgres://ibd-client-test:secret@%s/ibd-client-test?sslmode=disable", hostAndPort) + + // Kill container after 120 seconds + _ = resource.Expire(120) + + pool.MaxWait = 120 * time.Second + if err = pool.Retry(func() error { + db, err = sql.Open("postgres", databaseUrl) + if err != nil { + return err + } + return db.Ping() + }); err != nil { + log.Fatalf("Could not connect to database: %s", err) + } + + err = database.Migrate(context.Background(), databaseUrl) + if err != nil { + log.Fatalf("Could not migrate database: %s", err) + } + + defer func() { + if err := pool.Purge(resource); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + }() + + m.Run() +} + +func randStringRunes(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.IntN(len(letterRunes))] + } + return string(b) +} + +func addCookie(t *testing.T) (user, token string) { + t.Helper() + + // Randomly generate a user and token + user = randStringRunes(8) + token = randStringRunes(16) + + ciphertext, key, err := keys.Encrypt(context.Background(), new(kmsStub), "", []byte(token)) + require.NoError(t, err) + + tx, err := db.Begin() + require.NoError(t, err) + + var keyID uint + err = tx.QueryRow(` +INSERT INTO keys (kms_key_name, encrypted_key) + VALUES ('', $1) + RETURNING id; +`, key).Scan(&keyID) + require.NoError(t, err) + + _, err = tx.Exec(` +INSERT +INTO users (subject, encryption_key) +VALUES ($1, $2); +`, user, keyID) + require.NoError(t, err) + + _, err = tx.Exec(` +INSERT +INTO ibd_tokens (user_subject, token, encryption_key, expires_at) +VALUES ($1, $2, $3, $4);`, + user, + ciphertext, + keyID, + maxTime, + ) + require.NoError(t, err) + + err = tx.Commit() + require.NoError(t, err) + + return user, token +} - client := NewClient(new(emptyCookieSourceStub)) +func TestClient_getCookie(t *testing.T) { + t.Run("no cookies", func(t *testing.T) { + client := NewClient(db, new(kmsStub)) _, _, err := client.getCookie(context.Background(), nil) assert.ErrorIs(t, err, ErrNoAvailableCookies) }) t.Run("no cookies by subject", func(t *testing.T) { - t.Parallel() - - client := NewClient(new(emptyCookieSourceStub)) + client := NewClient(db, new(kmsStub)) subject := "test" _, _, err := client.getCookie(context.Background(), &subject) @@ -33,67 +158,44 @@ func TestClient_getCookie(t *testing.T) { }) t.Run("get any cookie", func(t *testing.T) { - t.Parallel() + _, token := addCookie(t) - client := NewClient(new(cookieSourceStub)) + client := NewClient(db, new(kmsStub)) - id, cookie, err := client.getCookie(context.Background(), nil) + _, cookie, err := client.getCookie(context.Background(), nil) require.NoError(t, err) - assert.Equal(t, uint(42), id) assert.Equal(t, cookieName, cookie.Name) - assert.Equal(t, "test-token", cookie.Value) + assert.Equal(t, token, cookie.Value) assert.Equal(t, "/", cookie.Path) - assert.Equal(t, time.Unix(0, 0), cookie.Expires) + assert.Equal(t, maxTime, cookie.Expires) assert.Equal(t, "investors.com", cookie.Domain) }) t.Run("get cookie by subject", func(t *testing.T) { - t.Parallel() + subject, token := addCookie(t) - client := NewClient(new(cookieSourceStub)) + client := NewClient(db, new(kmsStub)) - subject := "test" - id, cookie, err := client.getCookie(context.Background(), &subject) + _, cookie, err := client.getCookie(context.Background(), &subject) require.NoError(t, err) - assert.Equal(t, uint(42), id) assert.Equal(t, cookieName, cookie.Name) - assert.Equal(t, "test-token", cookie.Value) + assert.Equal(t, token, cookie.Value) assert.Equal(t, "/", cookie.Path) - assert.Equal(t, time.Unix(0, 0), cookie.Expires) + assert.Equal(t, maxTime, cookie.Expires) assert.Equal(t, "investors.com", cookie.Domain) }) } -type emptyCookieSourceStub struct{} - -func (c *emptyCookieSourceStub) GetAnyCookie(_ context.Context) (*database.IBDCookie, error) { - return nil, nil -} - -func (c *emptyCookieSourceStub) GetCookies(_ context.Context, _ string, _ bool) ([]database.IBDCookie, error) { - return nil, nil -} +type kmsStub struct{} -func (c *emptyCookieSourceStub) ReportCookieFailure(_ context.Context, _ uint) error { +func (k *kmsStub) Close() error { return nil } -var testCookie = database.IBDCookie{ - ID: 42, - Token: "test-token", - Expiry: time.Unix(0, 0), +func (k *kmsStub) Encrypt(_ context.Context, _ string, plaintext []byte) ([]byte, error) { + return plaintext, nil } -type cookieSourceStub struct{} - -func (c *cookieSourceStub) GetAnyCookie(_ context.Context) (*database.IBDCookie, error) { - return &testCookie, nil -} - -func (c *cookieSourceStub) GetCookies(_ context.Context, _ string, _ bool) ([]database.IBDCookie, error) { - return []database.IBDCookie{testCookie}, nil -} - -func (c *cookieSourceStub) ReportCookieFailure(_ context.Context, _ uint) error { - return nil +func (k *kmsStub) Decrypt(_ context.Context, _ string, ciphertext []byte) ([]byte, error) { + return ciphertext, nil } diff --git a/backend/internal/ibd/ibd50.go b/backend/internal/ibd/ibd50.go index ea02f82..52e28aa 100644 --- a/backend/internal/ibd/ibd50.go +++ b/backend/internal/ibd/ibd50.go @@ -9,6 +9,8 @@ import ( "net/http" "net/url" "strconv" + + "github.com/ansg191/ibd-trader-backend/internal/database" ) const ibd50Url = "https://research.investors.com/Services/SiteAjaxService.asmx/GetIBD50?sortcolumn1=%22ibd100rank%22&sortOrder1=%22asc%22&sortcolumn2=%22%22&sortOrder2=%22ASC%22" @@ -63,7 +65,7 @@ func (c *Client) GetIBD50(ctx context.Context) ([]*Stock, error) { // If there are less than 10 stocks in the IBD50 list, it's likely that authentication failed. if len(ibd50Resp.D.ETablesDataList) < 10 { // Report cookie failure to DB - if err = c.cookies.ReportCookieFailure(ctx, cookieId); err != nil { + if err = database.ReportCookieFailure(ctx, c.db, cookieId); err != nil { slog.Error("Failed to report cookie failure", "error", err) } return nil, errors.New("failed to get IBD50 list") diff --git a/backend/internal/ibd/search_test.go b/backend/internal/ibd/search_test.go index 99157cf..05e93dc 100644 --- a/backend/internal/ibd/search_test.go +++ b/backend/internal/ibd/search_test.go @@ -162,8 +162,6 @@ const emptySearchResponseJSON = ` }` func TestClient_Search(t *testing.T) { - t.Parallel() - tests := []struct { name string response string @@ -195,7 +193,11 @@ func TestClient_Search(t *testing.T) { tp := httpmock.NewMockTransport() tp.RegisterResponder("GET", searchUrl, httpmock.NewStringResponder(200, tt.response)) - client := NewClient(new(cookieSourceStub), transport.NewStandardTransport(&http.Client{Transport: tp})) + client := NewClient( + db, + new(kmsStub), + transport.NewStandardTransport(&http.Client{Transport: tp}), + ) tt.f(t, client) }) diff --git a/backend/internal/leader/manager/ibd/auth/auth.go b/backend/internal/leader/manager/ibd/auth/auth.go index 3419afd..9b5502d 100644 --- a/backend/internal/leader/manager/ibd/auth/auth.go +++ b/backend/internal/leader/manager/ibd/auth/auth.go @@ -20,13 +20,13 @@ const ( // Manager is responsible for sending authentication tasks to the workers. type Manager struct { queue taskqueue.TaskQueue[TaskInfo] - store database.UserStore + db database.Executor schedule cron.Schedule } func New( ctx context.Context, - store database.UserStore, + db database.Executor, rClient *redis.Client, schedule cron.Schedule, ) (*Manager, error) { @@ -43,7 +43,7 @@ func New( return &Manager{ queue: queue, - store: store, + db: db, schedule: schedule, }, nil } @@ -84,7 +84,7 @@ func (m *Manager) scrapeCookies(ctx context.Context, deadline time.Time) { defer cancel() // Get all users with IBD credentials - users, err := m.store.ListUsers(ctx, true) + users, err := database.ListUsers(ctx, m.db, true) if err != nil { slog.ErrorContext(ctx, "failed to get users", "error", err) return diff --git a/backend/internal/leader/manager/ibd/scrape/scrape.go b/backend/internal/leader/manager/ibd/scrape/scrape.go index e6cf490..870ce5e 100644 --- a/backend/internal/leader/manager/ibd/scrape/scrape.go +++ b/backend/internal/leader/manager/ibd/scrape/scrape.go @@ -23,7 +23,7 @@ const ( // Manager is responsible for sending scraping tasks to the workers. type Manager struct { client *ibd.Client - store database.StockStore + db database.Executor queue taskqueue.TaskQueue[TaskInfo] schedule cron.Schedule pubsub *redis.PubSub @@ -32,7 +32,7 @@ type Manager struct { func New( ctx context.Context, client *ibd.Client, - store database.StockStore, + db database.Executor, redis *redis.Client, schedule cron.Schedule, ) (*Manager, error) { @@ -49,7 +49,7 @@ func New( return &Manager{ client: client, - store: store, + db: db, queue: queue, schedule: schedule, pubsub: redis.Subscribe(ctx, Channel), @@ -107,7 +107,7 @@ func (m *Manager) scrapeIBD50(ctx context.Context, deadline time.Time) { for _, stock := range stocks { // Add stock to DB - err = m.store.AddStock(ctx, database.Stock{ + err = database.AddStock(ctx, m.db, database.Stock{ Symbol: stock.Symbol, Name: stock.Name, IBDUrl: stock.QuoteURL.String(), @@ -118,7 +118,7 @@ func (m *Manager) scrapeIBD50(ctx context.Context, deadline time.Time) { } // Add ranking to Db - err = m.store.AddRanking(ctx, stock.Symbol, int(stock.Rank), 0) + err = database.AddRanking(ctx, m.db, stock.Symbol, int(stock.Rank), 0) if err != nil { slog.ErrorContext(ctx, "failed to add ranking", "error", err) continue diff --git a/backend/internal/redis/taskqueue/queue_test.go b/backend/internal/redis/taskqueue/queue_test.go index aa817c5..774caa8 100644 --- a/backend/internal/redis/taskqueue/queue_test.go +++ b/backend/internal/redis/taskqueue/queue_test.go @@ -3,24 +3,64 @@ package taskqueue import ( "context" "errors" - "os" + "fmt" + "log" "testing" "time" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func getRedisClient() *redis.Client { - addr := os.Getenv("REDIS_ADDR") - if addr == "" { - addr = "localhost:6379" +var client *redis.Client + +func TestMain(m *testing.M) { + pool, err := dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not create pool: %s", err) + } + + err = pool.Client.Ping() + if err != nil { + log.Fatalf("Could not connect to Docker: %s", err) } - return redis.NewClient(&redis.Options{ - Addr: addr, + //resource, err := pool.Run("redis", "7", nil) + resource, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "redis", + Tag: "7", + }, func(config *docker.HostConfig) { + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{Name: "no"} }) + if err != nil { + log.Fatalf("Could not start resource: %s", err) + } + + _ = resource.Expire(60) + + if err = pool.Retry(func() error { + client = redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("localhost:%s", resource.GetPort("6379/tcp")), + }) + return client.Ping(context.Background()).Err() + }); err != nil { + log.Fatalf("Could not connect to redis: %s", err) + } + + defer func() { + if err = client.Close(); err != nil { + log.Printf("Could not close client: %s", err) + } + if err = pool.Purge(resource); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + }() + + m.Run() } func TestTaskQueue(t *testing.T) { @@ -28,11 +68,6 @@ func TestTaskQueue(t *testing.T) { t.Skip() } - client := getRedisClient() - defer func(client *redis.Client) { - _ = client.Close() - }(client) - lockTimeout := 100 * time.Millisecond tests := []struct { @@ -214,11 +249,6 @@ func TestTaskQueue_List(t *testing.T) { t.Skip() } - client := getRedisClient() - defer func(client *redis.Client) { - _ = client.Close() - }(client) - tests := []struct { name string f func(t *testing.T) @@ -378,16 +408,10 @@ func TestTaskQueue_List(t *testing.T) { } func TestTaskQueue_Return(t *testing.T) { - if testing.Short() { t.Skip() } - client := getRedisClient() - defer func(client *redis.Client) { - _ = client.Close() - }(client) - lockTimeout := 100 * time.Millisecond tests := []struct { diff --git a/backend/internal/server/idb/stock/v1/stock.go b/backend/internal/server/idb/stock/v1/stock.go index d30bde3..8afc2b1 100644 --- a/backend/internal/server/idb/stock/v1/stock.go +++ b/backend/internal/server/idb/stock/v1/stock.go @@ -22,11 +22,11 @@ const ScrapeOperationPrefix = "scrape" type Server struct { pb.UnimplementedStockServiceServer - db database.StockStore + db database.Executor queue taskqueue.TaskQueue[scrape.TaskInfo] } -func New(db database.StockStore, queue taskqueue.TaskQueue[scrape.TaskInfo]) *Server { +func New(db database.Executor, queue taskqueue.TaskQueue[scrape.TaskInfo]) *Server { return &Server{db: db, queue: queue} } diff --git a/backend/internal/server/idb/user/v1/user.go b/backend/internal/server/idb/user/v1/user.go index c100465..2f32e03 100644 --- a/backend/internal/server/idb/user/v1/user.go +++ b/backend/internal/server/idb/user/v1/user.go @@ -7,6 +7,7 @@ import ( pb "github.com/ansg191/ibd-trader-backend/api/gen/idb/user/v1" "github.com/ansg191/ibd-trader-backend/internal/database" "github.com/ansg191/ibd-trader-backend/internal/ibd" + "github.com/ansg191/ibd-trader-backend/internal/keys" "github.com/mennanov/fmutils" "google.golang.org/grpc/codes" @@ -17,26 +18,28 @@ import ( type Server struct { pb.UnimplementedUserServiceServer - user database.UserStore - cookie database.CookieSource - client *ibd.Client + db database.TransactionExecutor + kms keys.KeyManagementService + keyName string + client *ibd.Client } -func New(userStore database.UserStore, cookieStore database.CookieStore, client *ibd.Client) *Server { +func New(db database.TransactionExecutor, kms keys.KeyManagementService, keyName string, client *ibd.Client) *Server { return &Server{ - user: userStore, - cookie: cookieStore, - client: client, + db: db, + kms: kms, + keyName: keyName, + client: client, } } func (u *Server) CreateUser(ctx context.Context, request *pb.CreateUserRequest) (*pb.CreateUserResponse, error) { - err := u.user.AddUser(ctx, request.Subject) + err := database.AddUser(ctx, u.db, request.Subject) if err != nil { return nil, status.Errorf(codes.Internal, "unable to create user: %v", err) } - user, err := u.user.GetUser(ctx, request.Subject) + user, err := database.GetUser(ctx, u.db, request.Subject) if err != nil { return nil, status.Errorf(codes.Internal, "unable to get user: %v", err) } @@ -51,7 +54,7 @@ func (u *Server) CreateUser(ctx context.Context, request *pb.CreateUserRequest) } func (u *Server) GetUser(ctx context.Context, request *pb.GetUserRequest) (*pb.GetUserResponse, error) { - user, err := u.user.GetUser(ctx, request.Subject) + user, err := database.GetUser(ctx, u.db, request.Subject) if errors.Is(err, database.ErrUserNotFound) { return nil, status.New(codes.NotFound, "user not found").Err() } @@ -88,7 +91,7 @@ func (u *Server) UpdateUser(ctx context.Context, request *pb.UpdateUserRequest) (newUser.IbdPassword != existingUser.IbdPassword || newUser.IbdUsername != existingUser.IbdUsername) { // Update IBD creds - err = u.user.AddIBDCreds(ctx, newUser.Subject, *newUser.IbdUsername, *newUser.IbdPassword) + err = database.AddIBDCreds(ctx, u.db, u.kms, u.keyName, newUser.Subject, *newUser.IbdUsername, *newUser.IbdPassword) if err != nil { return nil, status.Errorf(codes.Internal, "unable to update user: %v", err) } @@ -119,7 +122,7 @@ func (u *Server) CheckIBDUsername(ctx context.Context, req *pb.CheckIBDUsernameR func (u *Server) AuthenticateUser(ctx context.Context, req *pb.AuthenticateUserRequest) (*pb.AuthenticateUserResponse, error) { // Check if user has cookies - cookies, err := u.cookie.GetCookies(ctx, req.Subject, false) + cookies, err := database.GetCookies(ctx, u.db, u.kms, req.Subject, false) if err != nil { return nil, status.Errorf(codes.Internal, "unable to get cookies: %v", err) } @@ -131,7 +134,7 @@ func (u *Server) AuthenticateUser(ctx context.Context, req *pb.AuthenticateUserR // Authenticate user // Get IBD creds - username, password, err := u.user.GetIBDCreds(ctx, req.Subject) + username, password, err := database.GetIBDCreds(ctx, u.db, u.kms, req.Subject) if errors.Is(err, database.ErrIBDCredsNotFound) { return nil, status.New(codes.NotFound, "User has no IDB creds").Err() } diff --git a/backend/internal/server/server.go b/backend/internal/server/server.go index 186d581..c525cfd 100644 --- a/backend/internal/server/server.go +++ b/backend/internal/server/server.go @@ -11,6 +11,7 @@ import ( upb "github.com/ansg191/ibd-trader-backend/api/gen/idb/user/v1" "github.com/ansg191/ibd-trader-backend/internal/database" "github.com/ansg191/ibd-trader-backend/internal/ibd" + "github.com/ansg191/ibd-trader-backend/internal/keys" "github.com/ansg191/ibd-trader-backend/internal/leader/manager/ibd/scrape" "github.com/ansg191/ibd-trader-backend/internal/redis/taskqueue" "github.com/ansg191/ibd-trader-backend/internal/server/idb/stock/v1" @@ -30,9 +31,11 @@ type Server struct { func New( ctx context.Context, port uint16, - db database.Database, + db database.TransactionExecutor, rClient *redis.Client, client *ibd.Client, + kms keys.KeyManagementService, + keyName string, ) (*Server, error) { scrapeQueue, err := taskqueue.New( ctx, @@ -45,7 +48,7 @@ func New( } s := grpc.NewServer() - upb.RegisterUserServiceServer(s, user.New(db, db, client)) + upb.RegisterUserServiceServer(s, user.New(db, kms, keyName, client)) spb.RegisterStockServiceServer(s, stock.New(db, scrapeQueue)) longrunningpb.RegisterOperationsServer(s, newOperationServer(scrapeQueue)) reflection.Register(s) diff --git a/backend/internal/worker/analyzer/analyzer.go b/backend/internal/worker/analyzer/analyzer.go index 79a35ee..ea8069e 100644 --- a/backend/internal/worker/analyzer/analyzer.go +++ b/backend/internal/worker/analyzer/analyzer.go @@ -24,7 +24,7 @@ func RunAnalyzer( ctx context.Context, redis *redis.Client, analyzer analyzer.Analyzer, - db database.StockStore, + db database.Executor, name string, ) error { queue, err := taskqueue.New( @@ -52,7 +52,7 @@ func waitForTask( ctx context.Context, queue taskqueue.TaskQueue[TaskInfo], analyzer analyzer.Analyzer, - db database.StockStore, + db database.Executor, ) { task, err := queue.Dequeue(ctx, lockTimeout, dequeueTimeout) if err != nil { @@ -111,8 +111,8 @@ func waitForTask( } } -func analyzeStock(ctx context.Context, a analyzer.Analyzer, db database.StockStore, id string) error { - info, err := db.GetStockInfo(ctx, id) +func analyzeStock(ctx context.Context, a analyzer.Analyzer, db database.Executor, id string) error { + info, err := database.GetStockInfo(ctx, db, id) if err != nil { return err } @@ -127,7 +127,7 @@ func analyzeStock(ctx context.Context, a analyzer.Analyzer, db database.StockSto return err } - return db.AddAnalysis(ctx, id, analysis) + return database.AddAnalysis(ctx, db, id, analysis) } type TaskInfo struct { diff --git a/backend/internal/worker/auth/auth.go b/backend/internal/worker/auth/auth.go index 1f591fe..579a180 100644 --- a/backend/internal/worker/auth/auth.go +++ b/backend/internal/worker/auth/auth.go @@ -2,12 +2,15 @@ package auth import ( "context" + "database/sql" + "errors" "fmt" "log/slog" "time" "github.com/ansg191/ibd-trader-backend/internal/database" "github.com/ansg191/ibd-trader-backend/internal/ibd" + "github.com/ansg191/ibd-trader-backend/internal/keys" "github.com/ansg191/ibd-trader-backend/internal/leader/manager/ibd/auth" "github.com/ansg191/ibd-trader-backend/internal/redis/taskqueue" @@ -23,8 +26,8 @@ func RunAuthScraper( ctx context.Context, client *ibd.Client, redis *redis.Client, - users database.UserStore, - cookies database.CookieStore, + db database.Executor, + kms keys.KeyManagementService, name string, ) error { queue, err := taskqueue.New( @@ -43,7 +46,7 @@ func RunAuthScraper( case <-ctx.Done(): return ctx.Err() default: - waitForTask(ctx, queue, client, users, cookies) + waitForTask(ctx, queue, client, db, kms) } } } @@ -52,8 +55,8 @@ func waitForTask( ctx context.Context, queue taskqueue.TaskQueue[auth.TaskInfo], client *ibd.Client, - users database.UserStore, - cookies database.CookieStore, + db database.Executor, + kms keys.KeyManagementService, ) { task, err := queue.Dequeue(ctx, lockTimeout, dequeueTimeout) if err != nil { @@ -69,7 +72,7 @@ func waitForTask( ch := make(chan error) defer close(ch) go func() { - ch <- scrapeCookies(ctx, client, users, cookies, task.Data.UserSubject) + ch <- scrapeCookies(ctx, client, db, kms, task.Data.UserSubject) }() ticker := time.NewTicker(lockTimeout / 5) @@ -116,15 +119,15 @@ func waitForTask( func scrapeCookies( ctx context.Context, client *ibd.Client, - users database.UserStore, - store database.CookieStore, + db database.Executor, + kms keys.KeyManagementService, user string, ) error { ctx, cancel := context.WithTimeout(ctx, lockTimeout) defer cancel() // Check if the user has valid cookies - done, err := hasValidCookies(ctx, store, user) + done, err := hasValidCookies(ctx, db, user) if err != nil { return fmt.Errorf("failed to check cookies: %w", err) } @@ -133,7 +136,7 @@ func scrapeCookies( } // Health check degraded cookies - done, err = healthCheckDegradedCookies(ctx, client, store, user) + done, err = healthCheckDegradedCookies(ctx, client, db, kms, user) if err != nil { return fmt.Errorf("failed to health check cookies: %w", err) } @@ -142,31 +145,39 @@ func scrapeCookies( } // No cookies are valid, so scrape new cookies - return scrapeNewCookies(ctx, client, users, store, user) + return scrapeNewCookies(ctx, client, db, kms, user) } -func hasValidCookies(ctx context.Context, store database.CookieStore, user string) (bool, error) { +func hasValidCookies(ctx context.Context, db database.Executor, user string) (bool, error) { // Check if the user has non-degraded cookies - cookies, err := store.GetCookies(ctx, user, false) + row := db.QueryRowContext(ctx, ` +SELECT 1 +FROM ibd_tokens +WHERE user_subject = $1 + AND expires_at > NOW() + AND degraded = FALSE;`, user) + + var exists bool + err := row.Scan(&exists) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } if err != nil { return false, fmt.Errorf("failed to get non-degraded cookies: %w", err) } - // If the user has non-degraded cookies, return true - if len(cookies) > 0 { - return true, nil - } - return false, nil + return true, nil } func healthCheckDegradedCookies( ctx context.Context, client *ibd.Client, - store database.CookieStore, + db database.Executor, + kms keys.KeyManagementService, user string, ) (bool, error) { // Check if the user has degraded cookies - cookies, err := store.GetCookies(ctx, user, true) + cookies, err := database.GetCookies(ctx, db, kms, user, true) if err != nil { return false, fmt.Errorf("failed to get degraded cookies: %w", err) } @@ -190,7 +201,7 @@ func healthCheckDegradedCookies( valid = true // Update the cookie - err = store.RepairCookie(ctx, cookie.ID) + err = database.RepairCookie(ctx, db, cookie.ID) if err != nil { slog.ErrorContext(ctx, "Failed to repair cookie", "error", err) } @@ -202,12 +213,12 @@ func healthCheckDegradedCookies( func scrapeNewCookies( ctx context.Context, client *ibd.Client, - users database.UserStore, - store database.CookieStore, + db database.Executor, + kms keys.KeyManagementService, user string, ) error { // Get the user's credentials - username, password, err := users.GetIBDCreds(ctx, user) + username, password, err := database.GetIBDCreds(ctx, db, kms, user) if err != nil { return fmt.Errorf("failed to get IBD credentials: %w", err) } @@ -219,7 +230,7 @@ func scrapeNewCookies( } // Store the cookie - err = store.AddCookie(ctx, user, cookie) + err = database.AddCookie(ctx, db, kms, user, cookie) if err != nil { return fmt.Errorf("failed to store cookie: %w", err) } diff --git a/backend/internal/worker/scraper/scraper.go b/backend/internal/worker/scraper/scraper.go index ec71d62..4788834 100644 --- a/backend/internal/worker/scraper/scraper.go +++ b/backend/internal/worker/scraper/scraper.go @@ -25,7 +25,7 @@ func RunScraper( ctx context.Context, redis *redis.Client, client *ibd.Client, - store database.StockStore, + db database.TransactionExecutor, name string, ) error { queue, err := taskqueue.New( @@ -55,7 +55,7 @@ func RunScraper( case <-ctx.Done(): return ctx.Err() default: - waitForTask(ctx, queue, aQueue, client, store) + waitForTask(ctx, queue, aQueue, client, db) } } } @@ -65,7 +65,7 @@ func waitForTask( queue taskqueue.TaskQueue[scrape.TaskInfo], aQueue taskqueue.TaskQueue[analyzer.TaskInfo], client *ibd.Client, - store database.StockStore, + db database.TransactionExecutor, ) { task, err := queue.Dequeue(ctx, lockTimeout, dequeueTimeout) if err != nil { @@ -80,7 +80,7 @@ func waitForTask( ch := make(chan error) go func() { defer close(ch) - ch <- scrapeUrl(ctx, client, store, aQueue, task.Data.Symbol) + ch <- scrapeUrl(ctx, client, db, aQueue, task.Data.Symbol) }() ticker := time.NewTicker(lockTimeout / 5) @@ -127,14 +127,14 @@ func waitForTask( func scrapeUrl( ctx context.Context, client *ibd.Client, - store database.StockStore, + db database.TransactionExecutor, aQueue taskqueue.TaskQueue[analyzer.TaskInfo], symbol string, ) error { ctx, cancel := context.WithTimeout(ctx, lockTimeout) defer cancel() - stockUrl, err := getStockUrl(ctx, store, client, symbol) + stockUrl, err := getStockUrl(ctx, db, client, symbol) if err != nil { return fmt.Errorf("failed to get stock url: %w", err) } @@ -146,7 +146,7 @@ func scrapeUrl( } // Add stock info to the database. - id, err := store.AddStockInfo(ctx, info) + id, err := database.AddStockInfo(ctx, db, info) if err != nil { return fmt.Errorf("failed to add stock info: %w", err) } @@ -162,9 +162,9 @@ func scrapeUrl( return nil } -func getStockUrl(ctx context.Context, store database.StockStore, client *ibd.Client, symbol string) (string, error) { +func getStockUrl(ctx context.Context, db database.TransactionExecutor, client *ibd.Client, symbol string) (string, error) { // Get the stock from the database. - stock, err := store.GetStock(ctx, symbol) + stock, err := database.GetStock(ctx, db, symbol) if err == nil { return stock.IBDUrl, nil } @@ -182,7 +182,7 @@ func getStockUrl(ctx context.Context, store database.StockStore, client *ibd.Cli } // Add the stock to the database. - err = store.AddStock(ctx, stock) + err = database.AddStock(ctx, db, stock) if err != nil { return "", fmt.Errorf("failed to add stock: %w", err) } diff --git a/backend/internal/worker/worker.go b/backend/internal/worker/worker.go index 3d7e9c8..6017fb7 100644 --- a/backend/internal/worker/worker.go +++ b/backend/internal/worker/worker.go @@ -12,6 +12,7 @@ import ( "github.com/ansg191/ibd-trader-backend/internal/analyzer" "github.com/ansg191/ibd-trader-backend/internal/database" "github.com/ansg191/ibd-trader-backend/internal/ibd" + "github.com/ansg191/ibd-trader-backend/internal/keys" "github.com/ansg191/ibd-trader-backend/internal/leader/manager" analyzer2 "github.com/ansg191/ibd-trader-backend/internal/worker/analyzer" "github.com/ansg191/ibd-trader-backend/internal/worker/auth" @@ -30,7 +31,8 @@ func StartWorker( ctx context.Context, ibdClient *ibd.Client, client *redis.Client, - db database.Database, + db database.TransactionExecutor, + kms keys.KeyManagementService, a analyzer.Analyzer, ) error { // Get the worker name. @@ -49,7 +51,7 @@ func StartWorker( return scraper.RunScraper(ctx, client, ibdClient, db, name) }) g.Go(func() error { - return auth.RunAuthScraper(ctx, ibdClient, client, db, db, name) + return auth.RunAuthScraper(ctx, ibdClient, client, db, kms, name) }) g.Go(func() error { return analyzer2.RunAnalyzer(ctx, client, a, db, name) |