Merge remote-tracking branch 'backend/main'

1 files changed, 182 insertions, 0 deletions

diff --git a/backend/internal/ibd/ibd50.go b/backend/internal/ibd/ibd50.go
new file mode 100644
index 0000000..52e28aa
--- /dev/null
+++ b/backend/internal/ibd/ibd50.go
@@ -0,0 +1,182 @@
+package ibd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/ansg191/ibd-trader-backend/internal/database"
+)
+
+const ibd50Url = "https://research.investors.com/Services/SiteAjaxService.asmx/GetIBD50?sortcolumn1=%22ibd100rank%22&sortOrder1=%22asc%22&sortcolumn2=%22%22&sortOrder2=%22ASC%22"
+
+// GetIBD50 returns the IBD50 list.
+func (c *Client) GetIBD50(ctx context.Context) ([]*Stock, error) {
+	// We cannot use the scraper here because scrapfly does not support
+	// Content-Type in GET requests.
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ibd50Url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	cookieId, cookie, err := c.getCookie(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.AddCookie(cookie)
+
+	req.Header.Add("content-type", "application/json; charset=utf-8")
+	// Add browser-emulating headers
+	req.Header.Add("accept", "*/*")
+	req.Header.Add("accept-language", "en-US,en;q=0.9")
+	req.Header.Add("newrelic", "eyJ2IjpbMCwxXSwiZCI6eyJ0eSI6IkJyb3dzZXIiLCJhYyI6IjMzOTYxMDYiLCJhcCI6IjEzODU5ODMwMDEiLCJpZCI6IjM1Zjk5NmM2MzNjYTViMWYiLCJ0ciI6IjM3ZmRhZmJlOGY2YjhmYTMwYWMzOTkzOGNlMmM0OWMxIiwidGkiOjE3MjIyNzg0NTk3MjUsInRrIjoiMTAyMjY4MSJ9fQ==")
+	req.Header.Add("priority", "u=1, i")
+	req.Header.Add("referer", "https://research.investors.com/stock-lists/ibd-50/")
+	req.Header.Add("sec-ch-ua", "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Google Chrome\";v=\"126\"")
+	req.Header.Add("sec-ch-ua-mobile", "?0")
+	req.Header.Add("sec-ch-ua-platform", "\"macOS\"")
+	req.Header.Add("sec-fetch-dest", "empty")
+	req.Header.Add("sec-fetch-mode", "cors")
+	req.Header.Add("sec-fetch-site", "same-origin")
+	req.Header.Add("traceparent", "00-37fdafbe8f6b8fa30ac39938ce2c49c1-35f996c633ca5b1f-01")
+	req.Header.Add("tracestate", "1022681@nr=0-1-3396106-1385983001-35f996c633ca5b1f----1722278459725")
+	req.Header.Add("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
+	req.Header.Add("x-newrelic-id", "VwUOV1dTDhABV1FRBgQOVVUF")
+	req.Header.Add("x-requested-with", "XMLHttpRequest")
+
+	resp, err := c.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func(Body io.ReadCloser) {
+		_ = Body.Close()
+	}(resp.Body)
+
+	var ibd50Resp getIBD50Response
+	if err = json.NewDecoder(resp.Body).Decode(&ibd50Resp); err != nil {
+		return nil, err
+	}
+
+	// If there are less than 10 stocks in the IBD50 list, it's likely that authentication failed.
+	if len(ibd50Resp.D.ETablesDataList) < 10 {
+		// Report cookie failure to DB
+		if err = database.ReportCookieFailure(ctx, c.db, cookieId); err != nil {
+			slog.Error("Failed to report cookie failure", "error", err)
+		}
+		return nil, errors.New("failed to get IBD50 list")
+	}
+
+	return ibd50Resp.ToStockList(), nil
+}
+
+type Stock struct {
+	Rank   int64
+	Symbol string
+	Name   string
+
+	QuoteURL *url.URL
+}
+
+type getIBD50Response struct {
+	D struct {
+		Type            *string `json:"__type"`
+		ETablesDataList []struct {
+			Rank                    string  `json:"Rank"`
+			Symbol                  string  `json:"Symbol"`
+			CompanyName             string  `json:"CompanyName"`
+			CompRating              *string `json:"CompRating"`
+			EPSRank                 *string `json:"EPSRank"`
+			RelSt                   *string `json:"RelSt"`
+			GrpStr                  *string `json:"GrpStr"`
+			Smr                     *string `json:"Smr"`
+			AccDis                  *string `json:"AccDis"`
+			SponRating              *string `json:"SponRating"`
+			Price                   *string `json:"Price"`
+			PriceClose              *string `json:"PriceClose"`
+			PriceChange             *string `json:"PriceChange"`
+			PricePerChange          *string `json:"PricePerChange"`
+			VolPerChange            *string `json:"VolPerChange"`
+			DailyVol                *string `json:"DailyVol"`
+			WeekHigh52              *string `json:"WeekHigh52"`
+			PerOffHigh              *string `json:"PerOffHigh"`
+			PERatio                 *string `json:"PERatio"`
+			DivYield                *string `json:"DivYield"`
+			LastQtrSalesPerChg      *string `json:"LastQtrSalesPerChg"`
+			LastQtrEpsPerChg        *string `json:"LastQtrEpsPerChg"`
+			ConsecQtrEpsGrt15       *string `json:"ConsecQtrEpsGrt15"`
+			CurQtrEpsEstPerChg      *string `json:"CurQtrEpsEstPerChg"`
+			CurYrEpsEstPerChg       *string `json:"CurYrEpsEstPerChg"`
+			PretaxMargin            *string `json:"PretaxMargin"`
+			ROE                     *string `json:"ROE"`
+			MgmtOwnsPer             *string `json:"MgmtOwnsPer"`
+			QuoteUrl                *string `json:"QuoteUrl"`
+			StockCheckupUrl         *string `json:"StockCheckupUrl"`
+			MarketsmithUrl          *string `json:"MarketsmithUrl"`
+			LeaderboardUrl          *string `json:"LeaderboardUrl"`
+			ChartAnalysisUrl        *string `json:"ChartAnalysisUrl"`
+			Ibd100NewEntryFlag      *string `json:"Ibd100NewEntryFlag"`
+			Ibd100UpInRankFlag      *string `json:"Ibd100UpInRankFlag"`
+			IbdBigCap20NewEntryFlag *string `json:"IbdBigCap20NewEntryFlag"`
+			CompDesc                *string `json:"CompDesc"`
+			NumberFunds             *string `json:"NumberFunds"`
+			GlobalRank              *string `json:"GlobalRank"`
+			EPSPriorQtr             *string `json:"EPSPriorQtr"`
+			QtrsFundIncrease        *string `json:"QtrsFundIncrease"`
+		} `json:"ETablesDataList"`
+		IBD50PdfUrl          *string `json:"IBD50PdfUrl"`
+		CAP20PdfUrl          *string `json:"CAP20PdfUrl"`
+		IBD50Date            *string `json:"IBD50Date"`
+		CAP20Date            *string `json:"CAP20Date"`
+		UpdatedDate          *string `json:"UpdatedDate"`
+		GetAllFlags          *string `json:"getAllFlags"`
+		Flag                 *int    `json:"flag"`
+		Message              *string `json:"Message"`
+		PaywallDesktopMarkup *string `json:"PaywallDesktopMarkup"`
+		PaywallMobileMarkup  *string `json:"PaywallMobileMarkup"`
+	} `json:"d"`
+}
+
+func (r getIBD50Response) ToStockList() (ibd []*Stock) {
+	ibd = make([]*Stock, 0, len(r.D.ETablesDataList))
+	for _, data := range r.D.ETablesDataList {
+		rank, err := strconv.ParseInt(data.Rank, 10, 64)
+		if err != nil {
+			slog.Error(
+				"Failed to parse Rank",
+				"error", err,
+				"rank", data.Rank,
+				"symbol", data.Symbol,
+				"name", data.CompanyName,
+			)
+			continue
+		}
+
+		var quoteUrl *url.URL
+		if data.QuoteUrl != nil {
+			quoteUrl, err = url.Parse(*data.QuoteUrl)
+			if err != nil {
+				slog.Error(
+					"Failed to parse QuoteUrl",
+					"error", err,
+					"quoteUrl", *data.QuoteUrl,
+					"rank", data.Rank,
+					"symbol", data.Symbol,
+					"name", data.CompanyName,
+				)
+			}
+		}
+
+		ibd = append(ibd, &Stock{
+			Rank:     rank,
+			Symbol:   data.Symbol,
+			Name:     data.CompanyName,
+			QuoteURL: quoteUrl,
+		})
+	}
+	return
+}