diff options
Diffstat (limited to 'backend/internal/ibd/ibd50.go')
| -rw-r--r-- | backend/internal/ibd/ibd50.go | 182 |
1 files changed, 182 insertions, 0 deletions
diff --git a/backend/internal/ibd/ibd50.go b/backend/internal/ibd/ibd50.go new file mode 100644 index 0000000..52e28aa --- /dev/null +++ b/backend/internal/ibd/ibd50.go @@ -0,0 +1,182 @@ +package ibd + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/url" + "strconv" + + "github.com/ansg191/ibd-trader-backend/internal/database" +) + +const ibd50Url = "https://research.investors.com/Services/SiteAjaxService.asmx/GetIBD50?sortcolumn1=%22ibd100rank%22&sortOrder1=%22asc%22&sortcolumn2=%22%22&sortOrder2=%22ASC%22" + +// GetIBD50 returns the IBD50 list. +func (c *Client) GetIBD50(ctx context.Context) ([]*Stock, error) { + // We cannot use the scraper here because scrapfly does not support + // Content-Type in GET requests. + req, err := http.NewRequestWithContext(ctx, http.MethodGet, ibd50Url, nil) + if err != nil { + return nil, err + } + + cookieId, cookie, err := c.getCookie(ctx, nil) + if err != nil { + return nil, err + } + req.AddCookie(cookie) + + req.Header.Add("content-type", "application/json; charset=utf-8") + // Add browser-emulating headers + req.Header.Add("accept", "*/*") + req.Header.Add("accept-language", "en-US,en;q=0.9") + req.Header.Add("newrelic", "eyJ2IjpbMCwxXSwiZCI6eyJ0eSI6IkJyb3dzZXIiLCJhYyI6IjMzOTYxMDYiLCJhcCI6IjEzODU5ODMwMDEiLCJpZCI6IjM1Zjk5NmM2MzNjYTViMWYiLCJ0ciI6IjM3ZmRhZmJlOGY2YjhmYTMwYWMzOTkzOGNlMmM0OWMxIiwidGkiOjE3MjIyNzg0NTk3MjUsInRrIjoiMTAyMjY4MSJ9fQ==") + req.Header.Add("priority", "u=1, i") + req.Header.Add("referer", "https://research.investors.com/stock-lists/ibd-50/") + req.Header.Add("sec-ch-ua", "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Google Chrome\";v=\"126\"") + req.Header.Add("sec-ch-ua-mobile", "?0") + req.Header.Add("sec-ch-ua-platform", "\"macOS\"") + req.Header.Add("sec-fetch-dest", "empty") + req.Header.Add("sec-fetch-mode", "cors") + req.Header.Add("sec-fetch-site", "same-origin") + req.Header.Add("traceparent", "00-37fdafbe8f6b8fa30ac39938ce2c49c1-35f996c633ca5b1f-01") + req.Header.Add("tracestate", "1022681@nr=0-1-3396106-1385983001-35f996c633ca5b1f----1722278459725") + req.Header.Add("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") + req.Header.Add("x-newrelic-id", "VwUOV1dTDhABV1FRBgQOVVUF") + req.Header.Add("x-requested-with", "XMLHttpRequest") + + resp, err := c.Do(req) + if err != nil { + return nil, err + } + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(resp.Body) + + var ibd50Resp getIBD50Response + if err = json.NewDecoder(resp.Body).Decode(&ibd50Resp); err != nil { + return nil, err + } + + // If there are less than 10 stocks in the IBD50 list, it's likely that authentication failed. + if len(ibd50Resp.D.ETablesDataList) < 10 { + // Report cookie failure to DB + if err = database.ReportCookieFailure(ctx, c.db, cookieId); err != nil { + slog.Error("Failed to report cookie failure", "error", err) + } + return nil, errors.New("failed to get IBD50 list") + } + + return ibd50Resp.ToStockList(), nil +} + +type Stock struct { + Rank int64 + Symbol string + Name string + + QuoteURL *url.URL +} + +type getIBD50Response struct { + D struct { + Type *string `json:"__type"` + ETablesDataList []struct { + Rank string `json:"Rank"` + Symbol string `json:"Symbol"` + CompanyName string `json:"CompanyName"` + CompRating *string `json:"CompRating"` + EPSRank *string `json:"EPSRank"` + RelSt *string `json:"RelSt"` + GrpStr *string `json:"GrpStr"` + Smr *string `json:"Smr"` + AccDis *string `json:"AccDis"` + SponRating *string `json:"SponRating"` + Price *string `json:"Price"` + PriceClose *string `json:"PriceClose"` + PriceChange *string `json:"PriceChange"` + PricePerChange *string `json:"PricePerChange"` + VolPerChange *string `json:"VolPerChange"` + DailyVol *string `json:"DailyVol"` + WeekHigh52 *string `json:"WeekHigh52"` + PerOffHigh *string `json:"PerOffHigh"` + PERatio *string `json:"PERatio"` + DivYield *string `json:"DivYield"` + LastQtrSalesPerChg *string `json:"LastQtrSalesPerChg"` + LastQtrEpsPerChg *string `json:"LastQtrEpsPerChg"` + ConsecQtrEpsGrt15 *string `json:"ConsecQtrEpsGrt15"` + CurQtrEpsEstPerChg *string `json:"CurQtrEpsEstPerChg"` + CurYrEpsEstPerChg *string `json:"CurYrEpsEstPerChg"` + PretaxMargin *string `json:"PretaxMargin"` + ROE *string `json:"ROE"` + MgmtOwnsPer *string `json:"MgmtOwnsPer"` + QuoteUrl *string `json:"QuoteUrl"` + StockCheckupUrl *string `json:"StockCheckupUrl"` + MarketsmithUrl *string `json:"MarketsmithUrl"` + LeaderboardUrl *string `json:"LeaderboardUrl"` + ChartAnalysisUrl *string `json:"ChartAnalysisUrl"` + Ibd100NewEntryFlag *string `json:"Ibd100NewEntryFlag"` + Ibd100UpInRankFlag *string `json:"Ibd100UpInRankFlag"` + IbdBigCap20NewEntryFlag *string `json:"IbdBigCap20NewEntryFlag"` + CompDesc *string `json:"CompDesc"` + NumberFunds *string `json:"NumberFunds"` + GlobalRank *string `json:"GlobalRank"` + EPSPriorQtr *string `json:"EPSPriorQtr"` + QtrsFundIncrease *string `json:"QtrsFundIncrease"` + } `json:"ETablesDataList"` + IBD50PdfUrl *string `json:"IBD50PdfUrl"` + CAP20PdfUrl *string `json:"CAP20PdfUrl"` + IBD50Date *string `json:"IBD50Date"` + CAP20Date *string `json:"CAP20Date"` + UpdatedDate *string `json:"UpdatedDate"` + GetAllFlags *string `json:"getAllFlags"` + Flag *int `json:"flag"` + Message *string `json:"Message"` + PaywallDesktopMarkup *string `json:"PaywallDesktopMarkup"` + PaywallMobileMarkup *string `json:"PaywallMobileMarkup"` + } `json:"d"` +} + +func (r getIBD50Response) ToStockList() (ibd []*Stock) { + ibd = make([]*Stock, 0, len(r.D.ETablesDataList)) + for _, data := range r.D.ETablesDataList { + rank, err := strconv.ParseInt(data.Rank, 10, 64) + if err != nil { + slog.Error( + "Failed to parse Rank", + "error", err, + "rank", data.Rank, + "symbol", data.Symbol, + "name", data.CompanyName, + ) + continue + } + + var quoteUrl *url.URL + if data.QuoteUrl != nil { + quoteUrl, err = url.Parse(*data.QuoteUrl) + if err != nil { + slog.Error( + "Failed to parse QuoteUrl", + "error", err, + "quoteUrl", *data.QuoteUrl, + "rank", data.Rank, + "symbol", data.Symbol, + "name", data.CompanyName, + ) + } + } + + ibd = append(ibd, &Stock{ + Rank: rank, + Symbol: data.Symbol, + Name: data.CompanyName, + QuoteURL: quoteUrl, + }) + } + return +} |