aboutsummaryrefslogtreecommitdiff
path: root/internal/reader/processor/processor.go
diff options
context:
space:
mode:
authorGravatar jvoisin <julien.voisin@dustri.org> 2024-03-30 23:01:02 +0100
committerGravatar Frédéric Guillot <f@miniflux.net> 2024-04-10 19:48:48 -0700
commitb205b5aad075dc89040231f87c79bec2a7ea60c7 (patch)
tree3e5426cd77fa27b64bc2808e724febddff3e7941 /internal/reader/processor/processor.go
parent4ab0d9422d14a9c9193feb4b2fa5c0a2a8360e8d (diff)
downloadv2-b205b5aad075dc89040231f87c79bec2a7ea60c7.tar.gz
v2-b205b5aad075dc89040231f87c79bec2a7ea60c7.tar.zst
v2-b205b5aad075dc89040231f87c79bec2a7ea60c7.zip
reader/processor: minimize the feed's entries html
Compress the html of feed entries before storing it. This should reduce the size of the database a bit, but more importantly, reduce the amount of data sent to clients minify being [stupidly fast](https://github.com/tdewolff/minify/?tab=readme-ov-file#performance), the performance impact should be in the noise level.
Diffstat (limited to 'internal/reader/processor/processor.go')
-rw-r--r--internal/reader/processor/processor.go20
1 files changed, 18 insertions, 2 deletions
diff --git a/internal/reader/processor/processor.go b/internal/reader/processor/processor.go
index eb2e2f9d..c92550d2 100644
--- a/internal/reader/processor/processor.go
+++ b/internal/reader/processor/processor.go
@@ -23,6 +23,8 @@ import (
"miniflux.app/v2/internal/storage"
"github.com/PuerkitoBio/goquery"
+ "github.com/tdewolff/minify/v2"
+ "github.com/tdewolff/minify/v2/html"
)
var (
@@ -36,6 +38,9 @@ var (
func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.User, forceRefresh bool) {
var filteredEntries model.Entries
+ minifier := minify.New()
+ minifier.AddFunc("text/html", html.Minify)
+
// Process older entries first
for i := len(feed.Entries) - 1; i >= 0; i-- {
entry := feed.Entries[i]
@@ -102,7 +107,11 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
)
} else if content != "" {
// We replace the entry content only if the scraper doesn't return any error.
- entry.Content = content
+ if minifiedHTML, err := minifier.String("text/html", content); err == nil {
+ entry.Content = minifiedHTML
+ } else {
+ entry.Content = content
+ }
}
}
@@ -180,6 +189,9 @@ func isAllowedEntry(feed *model.Feed, entry *model.Entry) bool {
// ProcessEntryWebPage downloads the entry web page and apply rewrite rules.
func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) error {
+ minifier := minify.New()
+ minifier.AddFunc("text/html", html.Minify)
+
startTime := time.Now()
websiteURL := getUrlFromEntry(feed, entry)
@@ -211,7 +223,11 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
}
if content != "" {
- entry.Content = content
+ if minifiedHTML, err := minifier.String("text/html", content); err == nil {
+ entry.Content = minifiedHTML
+ } else {
+ entry.Content = content
+ }
if user.ShowReadingTime {
entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed)
}