summaryrefslogtreecommitdiff
path: root/reader/scraper
diff options
context:
space:
mode:
Diffstat (limited to 'reader/scraper')
-rw-r--r--reader/scraper/scraper.go4
-rw-r--r--reader/scraper/scraper_test.go10
2 files changed, 7 insertions, 7 deletions
diff --git a/reader/scraper/scraper.go b/reader/scraper/scraper.go
index c0b968a7..f1b7e244 100644
--- a/reader/scraper/scraper.go
+++ b/reader/scraper/scraper.go
@@ -35,7 +35,7 @@ func Fetch(websiteURL, rules, userAgent string) (string, error) {
return "", errors.New("scraper: unable to download web page")
}
- if !isWhitelistedContentType(response.ContentType) {
+ if !isAllowedContentType(response.ContentType) {
return "", fmt.Errorf("scraper: this resource is not a HTML document (%s)", response.ContentType)
}
@@ -95,7 +95,7 @@ func getPredefinedScraperRules(websiteURL string) string {
return ""
}
-func isWhitelistedContentType(contentType string) bool {
+func isAllowedContentType(contentType string) bool {
contentType = strings.ToLower(contentType)
return strings.HasPrefix(contentType, "text/html") ||
strings.HasPrefix(contentType, "application/xhtml+xml")
diff --git a/reader/scraper/scraper_test.go b/reader/scraper/scraper_test.go
index 14e13ee8..5b1414df 100644
--- a/reader/scraper/scraper_test.go
+++ b/reader/scraper/scraper_test.go
@@ -39,7 +39,7 @@ func TestWhitelistedContentTypes(t *testing.T) {
}
for inputValue, expectedResult := range scenarios {
- actualResult := isWhitelistedContentType(inputValue)
+ actualResult := isAllowedContentType(inputValue)
if actualResult != expectedResult {
t.Errorf(`Unexpected result for content type whitelist, got "%v" instead of "%v"`, actualResult, expectedResult)
}
@@ -47,10 +47,10 @@ func TestWhitelistedContentTypes(t *testing.T) {
}
func TestSelectorRules(t *testing.T) {
- var ruleTestCases = map[string]string {
- "img.html": "article > img",
- "iframe.html": "article > iframe",
- "p.html": "article > p",
+ var ruleTestCases = map[string]string{
+ "img.html": "article > img",
+ "iframe.html": "article > iframe",
+ "p.html": "article > p",
}
for filename, rule := range ruleTestCases {