diff options
author | 2020-09-27 16:01:06 -0700 | |
---|---|---|
committer | 2020-09-27 20:04:48 -0700 | |
commit | c394a61a4e1fe395191262ed1ccaf70b90a71484 (patch) | |
tree | ca3080c4eeb29f3a01d47676be9521334e0696cd /reader/scraper/scraper.go | |
parent | 16b7b3bc3e4237abbacdf8695a685ca9a03dc5bb (diff) | |
download | v2-c394a61a4e1fe395191262ed1ccaf70b90a71484.tar.gz v2-c394a61a4e1fe395191262ed1ccaf70b90a71484.tar.zst v2-c394a61a4e1fe395191262ed1ccaf70b90a71484.zip |
Add Prometheus exporter
Diffstat (limited to 'reader/scraper/scraper.go')
-rw-r--r-- | reader/scraper/scraper.go | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/reader/scraper/scraper.go b/reader/scraper/scraper.go index c0b968a7..f1b7e244 100644 --- a/reader/scraper/scraper.go +++ b/reader/scraper/scraper.go @@ -35,7 +35,7 @@ func Fetch(websiteURL, rules, userAgent string) (string, error) { return "", errors.New("scraper: unable to download web page") } - if !isWhitelistedContentType(response.ContentType) { + if !isAllowedContentType(response.ContentType) { return "", fmt.Errorf("scraper: this resource is not a HTML document (%s)", response.ContentType) } @@ -95,7 +95,7 @@ func getPredefinedScraperRules(websiteURL string) string { return "" } -func isWhitelistedContentType(contentType string) bool { +func isAllowedContentType(contentType string) bool { contentType = strings.ToLower(contentType) return strings.HasPrefix(contentType, "text/html") || strings.HasPrefix(contentType, "application/xhtml+xml") |