package ibd import ( "context" "encoding/json" "errors" "io" "log/slog" "net/http" "net/url" "strconv" "github.com/ansg191/ibd-trader/backend/internal/database" ) const ibd50Url = "https://research.investors.com/Services/SiteAjaxService.asmx/GetIBD50?sortcolumn1=%22ibd100rank%22&sortOrder1=%22asc%22&sortcolumn2=%22%22&sortOrder2=%22ASC%22" // GetIBD50 returns the IBD50 list. func (c *Client) GetIBD50(ctx context.Context) ([]*Stock, error) { // We cannot use the scraper here because scrapfly does not support // Content-Type in GET requests. req, err := http.NewRequestWithContext(ctx, http.MethodGet, ibd50Url, nil) if err != nil { return nil, err } cookieId, cookie, err := c.getCookie(ctx, nil) if err != nil { return nil, err } req.AddCookie(cookie) req.Header.Add("content-type", "application/json; charset=utf-8") // Add browser-emulating headers req.Header.Add("accept", "*/*") req.Header.Add("accept-language", "en-US,en;q=0.9") req.Header.Add("newrelic", "eyJ2IjpbMCwxXSwiZCI6eyJ0eSI6IkJyb3dzZXIiLCJhYyI6IjMzOTYxMDYiLCJhcCI6IjEzODU5ODMwMDEiLCJpZCI6IjM1Zjk5NmM2MzNjYTViMWYiLCJ0ciI6IjM3ZmRhZmJlOGY2YjhmYTMwYWMzOTkzOGNlMmM0OWMxIiwidGkiOjE3MjIyNzg0NTk3MjUsInRrIjoiMTAyMjY4MSJ9fQ==") req.Header.Add("priority", "u=1, i") req.Header.Add("referer", "https://research.investors.com/stock-lists/ibd-50/") req.Header.Add("sec-ch-ua", "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Google Chrome\";v=\"126\"") req.Header.Add("sec-ch-ua-mobile", "?0") req.Header.Add("sec-ch-ua-platform", "\"macOS\"") req.Header.Add("sec-fetch-dest", "empty") req.Header.Add("sec-fetch-mode", "cors") req.Header.Add("sec-fetch-site", "same-origin") req.Header.Add("traceparent", "00-37fdafbe8f6b8fa30ac39938ce2c49c1-35f996c633ca5b1f-01") req.Header.Add("tracestate", "1022681@nr=0-1-3396106-1385983001-35f996c633ca5b1f----1722278459725") req.Header.Add("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36") req.Header.Add("x-newrelic-id", "VwUOV1dTDhABV1FRBgQOVVUF") req.Header.Add("x-requested-with", "XMLHttpRequest") resp, err := c.Do(req) if err != nil { return nil, err } defer func(Body io.ReadCloser) { _ = Body.Close() }(resp.Body) var ibd50Resp getIBD50Response if err = json.NewDecoder(resp.Body).Decode(&ibd50Resp); err != nil { return nil, err } // If there are less than 10 stocks in the IBD50 list, it's likely that authentication failed. if len(ibd50Resp.D.ETablesDataList) < 10 { // Report cookie failure to DB if err = database.ReportCookieFailure(ctx, c.db, cookieId); err != nil { slog.Error("Failed to report cookie failure", "error", err) } return nil, errors.New("failed to get IBD50 list") } return ibd50Resp.ToStockList(), nil } type Stock struct { Rank int64 Symbol string Name string QuoteURL *url.URL } type getIBD50Response struct { D struct { Type *string `json:"__type"` ETablesDataList []struct { Rank string `json:"Rank"` Symbol string `json:"Symbol"` CompanyName string `json:"CompanyName"` CompRating *string `json:"CompRating"` EPSRank *string `json:"EPSRank"` RelSt *string `json:"RelSt"` GrpStr *string `json:"GrpStr"` Smr *string `json:"Smr"` AccDis *string `json:"AccDis"` SponRating *string `json:"SponRating"` Price *string `json:"Price"` PriceClose *string `json:"PriceClose"` PriceChange *string `json:"PriceChange"` PricePerChange *string `json:"PricePerChange"` VolPerChange *string `json:"VolPerChange"` DailyVol *string `json:"DailyVol"` WeekHigh52 *string `json:"WeekHigh52"` PerOffHigh *string `json:"PerOffHigh"` PERatio *string `json:"PERatio"` DivYield *string `json:"DivYield"` LastQtrSalesPerChg *string `json:"LastQtrSalesPerChg"` LastQtrEpsPerChg *string `json:"LastQtrEpsPerChg"` ConsecQtrEpsGrt15 *string `json:"ConsecQtrEpsGrt15"` CurQtrEpsEstPerChg *string `json:"CurQtrEpsEstPerChg"` CurYrEpsEstPerChg *string `json:"CurYrEpsEstPerChg"` PretaxMargin *string `json:"PretaxMargin"` ROE *string `json:"ROE"` MgmtOwnsPer *string `json:"MgmtOwnsPer"` QuoteUrl *string `json:"QuoteUrl"` StockCheckupUrl *string `json:"StockCheckupUrl"` MarketsmithUrl *string `json:"MarketsmithUrl"` LeaderboardUrl *string `json:"LeaderboardUrl"` ChartAnalysisUrl *string `json:"ChartAnalysisUrl"` Ibd100NewEntryFlag *string `json:"Ibd100NewEntryFlag"` Ibd100UpInRankFlag *string `json:"Ibd100UpInRankFlag"` IbdBigCap20NewEntryFlag *string `json:"IbdBigCap20NewEntryFlag"` CompDesc *string `json:"CompDesc"` NumberFunds *string `json:"NumberFunds"` GlobalRank *string `json:"GlobalRank"` EPSPriorQtr *string `json:"EPSPriorQtr"` QtrsFundIncrease *string `json:"QtrsFundIncrease"` } `json:"ETablesDataList"` IBD50PdfUrl *string `json:"IBD50PdfUrl"` CAP20PdfUrl *string `json:"CAP20PdfUrl"` IBD50Date *string `json:"IBD50Date"` CAP20Date *string `json:"CAP20Date"` UpdatedDate *string `json:"UpdatedDate"` GetAllFlags *string `json:"getAllFlags"` Flag *int `json:"flag"` Message *string `json:"Message"` PaywallDesktopMarkup *string `json:"PaywallDesktopMarkup"` PaywallMobileMarkup *string `json:"PaywallMobileMarkup"` } `json:"d"` } func (r getIBD50Response) ToStockList() (ibd []*Stock) { ibd = make([]*Stock, 0, len(r.D.ETablesDataList)) for _, data := range r.D.ETablesDataList { rank, err := strconv.ParseInt(data.Rank, 10, 64) if err != nil { slog.Error( "Failed to parse Rank", "error", err, "rank", data.Rank, "symbol", data.Symbol, "name", data.CompanyName, ) continue } var quoteUrl *url.URL if data.QuoteUrl != nil { quoteUrl, err = url.Parse(*data.QuoteUrl) if err != nil { slog.Error( "Failed to parse QuoteUrl", "error", err, "quoteUrl", *data.QuoteUrl, "rank", data.Rank, "symbol", data.Symbol, "name", data.CompanyName, ) } } ibd = append(ibd, &Stock{ Rank: rank, Symbol: data.Symbol, Name: data.CompanyName, QuoteURL: quoteUrl, }) } return }