Skip to content

Commit

Permalink
reader/processor: minimize the feed's entries html
Browse files Browse the repository at this point in the history
Compress the html of feed entries before storing it. This should reduce the
size of the database a bit, but more importantly, reduce the amount of data
sent to clients

minify being [stupidly fast](https://github.com/tdewolff/minify/?tab=readme-ov-file#performance), the performance impact should be in the noise level.
  • Loading branch information
jvoisin authored and fguillot committed Apr 11, 2024
1 parent 4ab0d94 commit b205b5a
Showing 1 changed file with 18 additions and 2 deletions.
20 changes: 18 additions & 2 deletions internal/reader/processor/processor.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"miniflux.app/v2/internal/storage"

"github.com/PuerkitoBio/goquery"
"github.com/tdewolff/minify/v2"
"github.com/tdewolff/minify/v2/html"
)

var (
Expand All @@ -36,6 +38,9 @@ var (
func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.User, forceRefresh bool) {
var filteredEntries model.Entries

minifier := minify.New()
minifier.AddFunc("text/html", html.Minify)

// Process older entries first
for i := len(feed.Entries) - 1; i >= 0; i-- {
entry := feed.Entries[i]
Expand Down Expand Up @@ -102,7 +107,11 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
)
} else if content != "" {
// We replace the entry content only if the scraper doesn't return any error.
entry.Content = content
if minifiedHTML, err := minifier.String("text/html", content); err == nil {
entry.Content = minifiedHTML
} else {
entry.Content = content
}
}
}

Expand Down Expand Up @@ -180,6 +189,9 @@ func isAllowedEntry(feed *model.Feed, entry *model.Entry) bool {

// ProcessEntryWebPage downloads the entry web page and apply rewrite rules.
func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) error {
minifier := minify.New()
minifier.AddFunc("text/html", html.Minify)

startTime := time.Now()
websiteURL := getUrlFromEntry(feed, entry)

Expand Down Expand Up @@ -211,7 +223,11 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
}

if content != "" {
entry.Content = content
if minifiedHTML, err := minifier.String("text/html", content); err == nil {
entry.Content = minifiedHTML
} else {
entry.Content = content
}
if user.ShowReadingTime {
entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed)
}
Expand Down

0 comments on commit b205b5a

Please sign in to comment.