Modify IBD to accept various transport backends

This allows IBD to try using faster and cheaper transports first with
fallback to more reliable and expensive transports later.

1 files changed, 19 insertions, 1 deletions

diff --git a/backend/cmd/main.go b/backend/cmd/main.go
index c5104e8..16fc1ce 100644
--- a/backend/cmd/main.go
+++ b/backend/cmd/main.go
@@ -6,6 +6,7 @@ import (
 	"log"
 	"log/slog"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"time"
@@ -15,6 +16,8 @@ import (
 	"github.com/ansg191/ibd-trader-backend/internal/config"
 	"github.com/ansg191/ibd-trader-backend/internal/database"
 	"github.com/ansg191/ibd-trader-backend/internal/ibd"
+	"github.com/ansg191/ibd-trader-backend/internal/ibd/transport"
+	"github.com/ansg191/ibd-trader-backend/internal/ibd/transport/scrapfly"
 	"github.com/ansg191/ibd-trader-backend/internal/keys"
 	"github.com/ansg191/ibd-trader-backend/internal/leader/election"
 	"github.com/ansg191/ibd-trader-backend/internal/leader/manager"
@@ -75,7 +78,7 @@ func main() {
 	_ = auth
 
 	// Setup IBD client
-	client, err := ibd.NewClient(http.DefaultClient, cfg.IBD.APIKey, db, cfg.IBD.ProxyURL)
+	client, err := setupIBDClient(cfg, db)
 	if err != nil {
 		log.Fatal("Unable to setup IBD client: ", err)
 	}
@@ -143,6 +146,21 @@ func main() {
 	)
 }
 
+func setupIBDClient(cfg *config.Config, db database.Database) (*ibd.Client, error) {
+	pUrl, err := url.Parse(cfg.IBD.ProxyURL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse proxy URL: %w", err)
+	}
+	t := http.DefaultTransport.(*http.Transport).Clone()
+	t.Proxy = http.ProxyURL(pUrl)
+	transports := []transport.Transport{
+		&http.Client{Transport: t},                       // Default proxied transport
+		scrapfly.New(http.DefaultClient, cfg.IBD.APIKey), // Scrapfly transport
+	}
+	client := ibd.NewClient(transports, db)
+	return client, nil
+}
+
 func connectDB(logger *slog.Logger, cfg *config.Config) (database.Database, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()