From fd1fee852cb35fa0f5b0ed6dc0c23b4a6ce368c3 Mon Sep 17 00:00:00 2001 From: jvoisin Date: Mon, 11 Mar 2024 00:20:55 +0100 Subject: [PATCH 1/8] Simplify DomHelper.getVisibleElements Use a `filter` instead of a loop with an index. --- internal/ui/static/js/dom_helper.js | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/internal/ui/static/js/dom_helper.js b/internal/ui/static/js/dom_helper.js index fe0afcac3b2..352d6b03add 100644 --- a/internal/ui/static/js/dom_helper.js +++ b/internal/ui/static/js/dom_helper.js @@ -22,16 +22,8 @@ class DomHelper { } static getVisibleElements(selector) { - let elements = document.querySelectorAll(selector); - let result = []; - - for (let i = 0; i < elements.length; i++) { - if (this.isVisible(elements[i])) { - result.push(elements[i]); - } - } - - return result; + const elements = document.querySelectorAll(selector); + return [...elements].filter((element) => this.isVisible(element)); } static hasPassiveEventListenerOption() { From 74e4032ffc9faad4fec602f283a32d2af8dec47e Mon Sep 17 00:00:00 2001 From: jvoisin Date: Mon, 11 Mar 2024 17:23:20 +0100 Subject: [PATCH 2/8] Small refactor of app.js - replace a lot of `let` with `const` - inline some `querySelectorAll` calls - reduce the scope of some variables - use some ternaries where it makes sense - inline one-line functions --- internal/ui/static/js/app.js | 180 +++++++++++++++-------------------- 1 file changed, 76 insertions(+), 104 deletions(-) diff --git a/internal/ui/static/js/app.js b/internal/ui/static/js/app.js index 809c5c2204f..51a73e2ed3c 100644 --- a/internal/ui/static/js/app.js +++ b/internal/ui/static/js/app.js @@ -1,25 +1,21 @@ // OnClick attaches a listener to the elements that match the selector. function onClick(selector, callback, noPreventDefault) { - let elements = document.querySelectorAll(selector); - elements.forEach((element) => { + document.querySelectorAll(selector).forEach((element) => { element.onclick = (event) => { if (!noPreventDefault) { event.preventDefault(); } - callback(event); }; }); } function onAuxClick(selector, callback, noPreventDefault) { - let elements = document.querySelectorAll(selector); - elements.forEach((element) => { + document.querySelectorAll(selector).forEach((element) => { element.onauxclick = (event) => { if (!noPreventDefault) { event.preventDefault(); } - callback(event); }; }); @@ -28,22 +24,18 @@ function onAuxClick(selector, callback, noPreventDefault) { // make logo element as button on mobile layout function checkMenuToggleModeByLayout() { const logoElement = document.querySelector(".logo"); - const homePageLinkElement = document.querySelector(".logo > a"); if (!logoElement) return; - const logoToggleButtonLabel = logoElement.getAttribute("data-toggle-button-label"); - const navMenuElement = document.getElementById("header-menu"); - const navMenuElementIsExpanded = navMenuElement.classList.contains("js-menu-show"); + const homePageLinkElement = document.querySelector(".logo > a"); if (document.documentElement.clientWidth < 620) { + const navMenuElement = document.getElementById("header-menu"); + const navMenuElementIsExpanded = navMenuElement.classList.contains("js-menu-show"); + const logoToggleButtonLabel = logoElement.getAttribute("data-toggle-button-label"); logoElement.setAttribute("role", "button"); logoElement.setAttribute("tabindex", "0"); logoElement.setAttribute("aria-label", logoToggleButtonLabel); - if (navMenuElementIsExpanded) { - logoElement.setAttribute("aria-expanded", "true"); - } else { - logoElement.setAttribute("aria-expanded", "false"); - } + logoElement.setAttribute("aria-expanded", navMenuElementIsExpanded?"true":"false"); homePageLinkElement.setAttribute("tabindex", "-1"); } else { logoElement.removeAttribute("role"); @@ -55,24 +47,15 @@ function checkMenuToggleModeByLayout() { } function fixVoiceOverDetailsSummaryBug() { - const detailsElements = document.querySelectorAll("details"); - detailsElements.forEach((details) => { + document.querySelectorAll("details").forEach((details) => { const summaryElement = details.querySelector("summary"); summaryElement.setAttribute("role", "button"); - setSummaryAriaExpandedByDetails(details, summaryElement); + summaryElement.setAttribute("aria-expanded", details.open? "true": "false"); details.addEventListener("toggle", () => { - setSummaryAriaExpandedByDetails(details, summaryElement); + summaryElement.setAttribute("aria-expanded", details.open? "true": "false"); }); }); - - function setSummaryAriaExpandedByDetails(details, summary) { - if (details.open) { - summary.setAttribute("aria-expanded", "true"); - } else { - summary.setAttribute("aria-expanded", "false"); - } - } } // Show and hide the main menu on mobile devices. @@ -85,8 +68,8 @@ function toggleMainMenu(event) { event.preventDefault(); } - let menu = document.querySelector(".header nav ul"); - let menuToggleButton = document.querySelector(".logo"); + const menu = document.querySelector(".header nav ul"); + const menuToggleButton = document.querySelector(".logo"); if (menu.classList.contains("js-menu-show")) { menu.classList.remove("js-menu-show"); menuToggleButton.setAttribute("aria-expanded", false); @@ -98,7 +81,7 @@ function toggleMainMenu(event) { // Handle click events for the main menu (
  • and ). function onClickMainMenuListItem(event) { - let element = event.target; + const element = event.target; if (element.tagName === "A") { window.location.href = element.getAttribute("href"); @@ -109,11 +92,9 @@ function onClickMainMenuListItem(event) { // Change the button label when the page is loading. function handleSubmitButtons() { - let elements = document.querySelectorAll("form"); - elements.forEach((element) => { + document.querySelectorAll("form").forEach((element) => { element.onsubmit = () => { - let button = element.querySelector("button"); - + const button = element.querySelector("button"); if (button) { button.textContent = button.dataset.labelLoading; button.disabled = true; @@ -124,7 +105,7 @@ function handleSubmitButtons() { // Show modal dialog with the list of keyboard shortcuts. function showKeyboardShortcuts() { - let template = document.getElementById("keyboard-shortcuts"); + const template = document.getElementById("keyboard-shortcuts"); if (template !== null) { ModalHandler.open(template.content, "dialog-title"); } @@ -132,8 +113,8 @@ function showKeyboardShortcuts() { // Mark as read visible items of the current page. function markPageAsRead() { - let items = DomHelper.getVisibleElements(".items .item"); - let entryIDs = []; + const items = DomHelper.getVisibleElements(".items .item"); + const entryIDs = []; items.forEach((element) => { element.classList.add("item-status-read"); @@ -144,7 +125,7 @@ function markPageAsRead() { updateEntriesStatus(entryIDs, "read", () => { // Make sure the Ajax request reach the server before we reload the page. - let element = document.querySelector(":is(a, button)[data-action=markPageAsRead]"); + const element = document.querySelector(":is(a, button)[data-action=markPageAsRead]"); let showOnlyUnread = false; if (element) { showOnlyUnread = element.dataset.showOnlyUnread || false; @@ -167,8 +148,8 @@ function markPageAsRead() { * @param {boolean} setToRead */ function handleEntryStatus(item, element, setToRead) { - let toasting = !element; - let currentEntry = findEntry(element); + const toasting = !element; + const currentEntry = findEntry(element); if (currentEntry) { if (!setToRead || currentEntry.querySelector(":is(a, button)[data-toggle-status]").dataset.value == "unread") { toggleEntryStatus(currentEntry, toasting); @@ -188,11 +169,11 @@ function handleEntryStatus(item, element, setToRead) { // Change the entry status to the opposite value. function toggleEntryStatus(element, toasting) { - let entryID = parseInt(element.dataset.id, 10); - let link = element.querySelector(":is(a, button)[data-toggle-status]"); + const entryID = parseInt(element.dataset.id, 10); + const link = element.querySelector(":is(a, button)[data-toggle-status]"); - let currentStatus = link.dataset.value; - let newStatus = currentStatus === "read" ? "unread" : "read"; + const currentStatus = link.dataset.value; + const newStatus = currentStatus === "read" ? "unread" : "read"; link.querySelector("span").textContent = link.dataset.labelLoading; updateEntriesStatus([entryID], newStatus, () => { @@ -228,15 +209,14 @@ function markEntryAsRead(element) { element.classList.remove("item-status-unread"); element.classList.add("item-status-read"); - let entryID = parseInt(element.dataset.id, 10); + const entryID = parseInt(element.dataset.id, 10); updateEntriesStatus([entryID], "read"); } } // Send the Ajax request to refresh all feeds in the background function handleRefreshAllFeeds() { - let url = document.body.dataset.refreshAllFeedsUrl; - + const url = document.body.dataset.refreshAllFeedsUrl; if (url) { window.location.href = url; } @@ -244,8 +224,8 @@ function handleRefreshAllFeeds() { // Send the Ajax request to change entries statuses. function updateEntriesStatus(entryIDs, status, callback) { - let url = document.body.dataset.entriesStatusUrl; - let request = new RequestBuilder(url); + const url = document.body.dataset.entriesStatusUrl; + const request = new RequestBuilder(url); request.withBody({ entry_ids: entryIDs, status: status }); request.withCallback((resp) => { resp.json().then(count => { @@ -265,8 +245,8 @@ function updateEntriesStatus(entryIDs, status, callback) { // Handle save entry from list view and entry view. function handleSaveEntry(element) { - let toasting = !element; - let currentEntry = findEntry(element); + const toasting = !element; + const currentEntry = findEntry(element); if (currentEntry) { saveEntry(currentEntry.querySelector(":is(a, button)[data-save-entry]"), toasting); } @@ -274,22 +254,18 @@ function handleSaveEntry(element) { // Send the Ajax request to save an entry. function saveEntry(element, toasting) { - if (!element) { - return; - } - - if (element.dataset.completed) { + if (!element || element.dataset.completed) { return; } element.innerHTML = '' + element.dataset.labelLoading + ''; - let request = new RequestBuilder(element.dataset.saveUrl); + const request = new RequestBuilder(element.dataset.saveUrl); request.withCallback(() => { element.innerHTML = '' + element.dataset.labelDone + ''; element.dataset.completed = true; if (toasting) { - let iconElement = document.querySelector("template#icon-save"); + const iconElement = document.querySelector("template#icon-save"); showToast(element.dataset.toastDone, iconElement); } }); @@ -298,8 +274,8 @@ function saveEntry(element, toasting) { // Handle bookmark from the list view and entry view. function handleBookmark(element) { - let toasting = !element; - let currentEntry = findEntry(element); + const toasting = !element; + const currentEntry = findEntry(element); if (currentEntry) { toggleBookmark(currentEntry, toasting); } @@ -307,21 +283,19 @@ function handleBookmark(element) { // Send the Ajax request and change the icon when bookmarking an entry. function toggleBookmark(parentElement, toasting) { - let element = parentElement.querySelector(":is(a, button)[data-toggle-bookmark]"); + const element = parentElement.querySelector(":is(a, button)[data-toggle-bookmark]"); if (!element) { return; } element.innerHTML = '' + element.dataset.labelLoading + ''; - let request = new RequestBuilder(element.dataset.bookmarkUrl); + const request = new RequestBuilder(element.dataset.bookmarkUrl); request.withCallback(() => { - - let currentStarStatus = element.dataset.value; - let newStarStatus = currentStarStatus === "star" ? "unstar" : "star"; + const currentStarStatus = element.dataset.value; + const newStarStatus = currentStarStatus === "star" ? "unstar" : "star"; let iconElement, label; - if (currentStarStatus === "star") { iconElement = document.querySelector("template#icon-star"); label = element.dataset.labelStar; @@ -348,15 +322,15 @@ function handleFetchOriginalContent() { return; } - let element = document.querySelector(":is(a, button)[data-fetch-content-entry]"); + const element = document.querySelector(":is(a, button)[data-fetch-content-entry]"); if (!element) { return; } - let previousElement = element.cloneNode(true); + const previousElement = element.cloneNode(true); element.innerHTML = '' + element.dataset.labelLoading + ''; - let request = new RequestBuilder(element.dataset.fetchContentUrl); + const request = new RequestBuilder(element.dataset.fetchContentUrl); request.withCallback((response) => { element.textContent = ''; element.appendChild(previousElement); @@ -364,7 +338,7 @@ function handleFetchOriginalContent() { response.json().then((data) => { if (data.hasOwnProperty("content") && data.hasOwnProperty("reading_time")) { document.querySelector(".entry-content").innerHTML = data.content; - let entryReadingtimeElement = document.querySelector(".entry-reading-time"); + const entryReadingtimeElement = document.querySelector(".entry-reading-time"); if (entryReadingtimeElement) { entryReadingtimeElement.innerHTML = data.reading_time; } @@ -375,7 +349,7 @@ function handleFetchOriginalContent() { } function openOriginalLink(openLinkInCurrentTab) { - let entryLink = document.querySelector(".entry h1 a"); + const entryLink = document.querySelector(".entry h1 a"); if (entryLink !== null) { if (openLinkInCurrentTab) { window.location.href = entryLink.getAttribute("href"); @@ -385,11 +359,11 @@ function openOriginalLink(openLinkInCurrentTab) { return; } - let currentItemOriginalLink = document.querySelector(".current-item :is(a, button)[data-original-link]"); + const currentItemOriginalLink = document.querySelector(".current-item :is(a, button)[data-original-link]"); if (currentItemOriginalLink !== null) { DomHelper.openNewTab(currentItemOriginalLink.getAttribute("href")); - let currentItem = document.querySelector(".current-item"); + const currentItem = document.querySelector(".current-item"); // If we are not on the list of starred items, move to the next item if (document.location.href != document.querySelector(':is(a, button)[data-page=starred]').href) { goToListItem(1); @@ -400,7 +374,7 @@ function openOriginalLink(openLinkInCurrentTab) { function openCommentLink(openLinkInCurrentTab) { if (!isListView()) { - let entryLink = document.querySelector(":is(a, button)[data-comments-link]"); + const entryLink = document.querySelector(":is(a, button)[data-comments-link]"); if (entryLink !== null) { if (openLinkInCurrentTab) { window.location.href = entryLink.getAttribute("href"); @@ -410,7 +384,7 @@ function openCommentLink(openLinkInCurrentTab) { return; } } else { - let currentItemCommentsLink = document.querySelector(".current-item :is(a, button)[data-comments-link]"); + const currentItemCommentsLink = document.querySelector(".current-item :is(a, button)[data-comments-link]"); if (currentItemCommentsLink !== null) { DomHelper.openNewTab(currentItemCommentsLink.getAttribute("href")); } @@ -418,18 +392,18 @@ function openCommentLink(openLinkInCurrentTab) { } function openSelectedItem() { - let currentItemLink = document.querySelector(".current-item .item-title a"); + const currentItemLink = document.querySelector(".current-item .item-title a"); if (currentItemLink !== null) { window.location.href = currentItemLink.getAttribute("href"); } } function unsubscribeFromFeed() { - let unsubscribeLinks = document.querySelectorAll("[data-action=remove-feed]"); + const unsubscribeLinks = document.querySelectorAll("[data-action=remove-feed]"); if (unsubscribeLinks.length === 1) { - let unsubscribeLink = unsubscribeLinks[0]; + const unsubscribeLink = unsubscribeLinks[0]; - let request = new RequestBuilder(unsubscribeLink.dataset.url); + const request = new RequestBuilder(unsubscribeLink.dataset.url); request.withCallback(() => { if (unsubscribeLink.dataset.redirectUrl) { window.location.href = unsubscribeLink.dataset.redirectUrl; @@ -446,7 +420,7 @@ function unsubscribeFromFeed() { * @param {boolean} fallbackSelf Refresh actual page if the page is not found. */ function goToPage(page, fallbackSelf) { - let element = document.querySelector(":is(a, button)[data-page=" + page + "]"); + const element = document.querySelector(":is(a, button)[data-page=" + page + "]"); if (element) { document.location.href = element.href; @@ -481,12 +455,12 @@ function goToFeedOrFeeds() { function goToFeed() { if (isEntry()) { - let feedAnchor = document.querySelector("span.entry-website a"); + const feedAnchor = document.querySelector("span.entry-website a"); if (feedAnchor !== null) { window.location.href = feedAnchor.href; } } else { - let currentItemFeed = document.querySelector(".current-item :is(a, button)[data-feed-link]"); + const currentItemFeed = document.querySelector(".current-item :is(a, button)[data-feed-link]"); if (currentItemFeed !== null) { window.location.href = currentItemFeed.getAttribute("href"); } @@ -497,7 +471,7 @@ function goToFeed() { * @param {number} offset How many items to jump for focus. */ function goToListItem(offset) { - let items = DomHelper.getVisibleElements(".items .item"); + const items = DomHelper.getVisibleElements(".items .item"); if (items.length === 0) { return; } @@ -512,7 +486,8 @@ function goToListItem(offset) { if (items[i].classList.contains("current-item")) { items[i].classList.remove("current-item"); - let item = items[(i + offset + items.length) % items.length]; + const index = (i + offset + items.length) % items.length; + const item = items[index]; item.classList.add("current-item"); DomHelper.scrollPageTo(item); @@ -524,7 +499,7 @@ function goToListItem(offset) { } function scrollToCurrentItem() { - let currentItem = document.querySelector(".current-item"); + const currentItem = document.querySelector(".current-item"); if (currentItem !== null) { DomHelper.scrollPageTo(currentItem, true); } @@ -543,15 +518,14 @@ function incrementUnreadCounter(n) { } function updateUnreadCounterValue(callback) { - let counterElements = document.querySelectorAll("span.unread-counter"); - counterElements.forEach((element) => { - let oldValue = parseInt(element.textContent, 10); + document.querySelectorAll("span.unread-counter").forEach((element) => { + const oldValue = parseInt(element.textContent, 10); element.innerHTML = callback(oldValue); }); if (window.location.href.endsWith('/unread')) { - let oldValue = parseInt(document.title.split('(')[1], 10); - let newValue = callback(oldValue); + const oldValue = parseInt(document.title.split('(')[1], 10); + const newValue = callback(oldValue); document.title = document.title.replace( /(.*?)\(\d+\)(.*?)/, @@ -574,12 +548,10 @@ function findEntry(element) { if (isListView()) { if (element) { return element.closest(".item"); - } else { - return document.querySelector(".current-item"); } - } else { - return document.querySelector(".entry"); + return document.querySelector(".current-item"); } + return document.querySelector(".entry"); } function handleConfirmationMessage(linkElement, callback) { @@ -589,11 +561,11 @@ function handleConfirmationMessage(linkElement, callback) { linkElement.style.display = "none"; - let containerElement = linkElement.parentNode; - let questionElement = document.createElement("span"); + const containerElement = linkElement.parentNode; + const questionElement = document.createElement("span"); function createLoadingElement() { - let loadingElement = document.createElement("span"); + const loadingElement = document.createElement("span"); loadingElement.className = "loading"; loadingElement.appendChild(document.createTextNode(linkElement.dataset.labelLoading)); @@ -601,7 +573,7 @@ function handleConfirmationMessage(linkElement, callback) { containerElement.appendChild(loadingElement); } - let yesElement = document.createElement("button"); + const yesElement = document.createElement("button"); yesElement.appendChild(document.createTextNode(linkElement.dataset.labelYes)); yesElement.onclick = (event) => { event.preventDefault(); @@ -611,7 +583,7 @@ function handleConfirmationMessage(linkElement, callback) { callback(linkElement.dataset.url, linkElement.dataset.redirectUrl); }; - let noElement = document.createElement("button"); + const noElement = document.createElement("button"); noElement.appendChild(document.createTextNode(linkElement.dataset.labelNo)); noElement.onclick = (event) => { event.preventDefault(); @@ -674,7 +646,7 @@ function handlePlayerProgressionSave(playerElement) { currentPositionInSeconds <= (lastKnownPositionInSeconds - recordInterval) ) { playerElement.dataset.lastPosition = currentPositionInSeconds.toString(); - let request = new RequestBuilder(playerElement.dataset.saveUrl); + const request = new RequestBuilder(playerElement.dataset.saveUrl); request.withBody({ progression: currentPositionInSeconds }); request.execute(); } @@ -684,13 +656,13 @@ function handlePlayerProgressionSave(playerElement) { * handle new share entires and already shared entries */ function handleShare() { - let link = document.querySelector(':is(a, button)[data-share-status]'); - let title = document.querySelector("body > main > section > header > h1 > a"); + const link = document.querySelector(':is(a, button)[data-share-status]'); + const title = document.querySelector("body > main > section > header > h1 > a"); if (link.dataset.shareStatus === "shared") { checkShareAPI(title, link.href); } if (link.dataset.shareStatus === "share") { - let request = new RequestBuilder(link.href); + const request = new RequestBuilder(link.href); request.withCallback((r) => { checkShareAPI(title, r.url); }); @@ -721,7 +693,7 @@ function checkShareAPI(title, url) { } function getCsrfToken() { - let element = document.querySelector("body[data-csrf-token]"); + const element = document.querySelector("body[data-csrf-token]"); if (element !== null) { return element.dataset.csrfToken; } From 9c8a7dfffe2f4596dcbde2c923a7539914bb252f Mon Sep 17 00:00:00 2001 From: jvoisin Date: Mon, 11 Mar 2024 23:03:23 +0100 Subject: [PATCH 3/8] Make use of HashFromBytes everywhere It feels a bit silly to have a function and to not make use of it. --- internal/crypto/crypto.go | 3 +-- internal/ui/static/static.go | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 0b0ab6c6a75..c99beeb8a03 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -17,8 +17,7 @@ import ( // HashFromBytes returns a SHA-256 checksum of the input. func HashFromBytes(value []byte) string { - sum := sha256.Sum256(value) - return fmt.Sprintf("%x", sum) + return fmt.Sprintf("%x", sha256.Sum256(value)) } // Hash returns a SHA-256 checksum of a string. diff --git a/internal/ui/static/static.go b/internal/ui/static/static.go index a3deb6d9d95..c417d23f952 100644 --- a/internal/ui/static/static.go +++ b/internal/ui/static/static.go @@ -5,10 +5,11 @@ package static // import "miniflux.app/v2/internal/ui/static" import ( "bytes" - "crypto/sha256" "embed" "fmt" + "miniflux.app/v2/internal/crypto" + "github.com/tdewolff/minify/v2" "github.com/tdewolff/minify/v2/css" "github.com/tdewolff/minify/v2/js" @@ -48,7 +49,7 @@ func CalculateBinaryFileChecksums() error { return err } - binaryFileChecksums[dirEntry.Name()] = fmt.Sprintf("%x", sha256.Sum256(data)) + binaryFileChecksums[dirEntry.Name()] = crypto.HashFromBytes(data) } return nil @@ -102,7 +103,7 @@ func GenerateStylesheetsBundles() error { } StylesheetBundles[bundle] = minifiedData - StylesheetBundleChecksums[bundle] = fmt.Sprintf("%x", sha256.Sum256(minifiedData)) + StylesheetBundleChecksums[bundle] = crypto.HashFromBytes(minifiedData) } return nil @@ -166,7 +167,7 @@ func GenerateJavascriptBundles() error { } JavascriptBundles[bundle] = minifiedData - JavascriptBundleChecksums[bundle] = fmt.Sprintf("%x", sha256.Sum256(minifiedData)) + JavascriptBundleChecksums[bundle] = crypto.HashFromBytes(minifiedData) } return nil From 5bcb37901c60463b27e1211e0f68295f213b19e6 Mon Sep 17 00:00:00 2001 From: jvoisin Date: Mon, 11 Mar 2024 23:24:46 +0100 Subject: [PATCH 4/8] Use crypto.GenerateRandomBytes instead of doing it by hand This makes the code a bit shorter, and properly handle cryptographic error conditions. --- internal/config/options.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/config/options.go b/internal/config/options.go index dc3d7063e29..483192f9946 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -4,12 +4,12 @@ package config // import "miniflux.app/v2/internal/config" import ( - "crypto/rand" "fmt" "sort" "strings" "time" + "miniflux.app/v2/internal/crypto" "miniflux.app/v2/internal/version" ) @@ -171,9 +171,6 @@ type Options struct { // NewOptions returns Options with default values. func NewOptions() *Options { - randomKey := make([]byte, 16) - rand.Read(randomKey) - return &Options{ HTTPS: defaultHTTPS, logFile: defaultLogFile, @@ -242,7 +239,7 @@ func NewOptions() *Options { metricsPassword: defaultMetricsPassword, watchdog: defaultWatchdog, invidiousInstance: defaultInvidiousInstance, - proxyPrivateKey: randomKey, + proxyPrivateKey: crypto.GenerateRandomBytes(16), webAuthn: defaultWebAuthn, } } From d3a85b049b14d4a4ddd6b813134b2abd45fe5e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Mon, 11 Mar 2024 17:23:14 -0700 Subject: [PATCH 5/8] jsminifier: set JavaScript version --- internal/ui/static/static.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/ui/static/static.go b/internal/ui/static/static.go index c417d23f952..fd653b81035 100644 --- a/internal/ui/static/static.go +++ b/internal/ui/static/static.go @@ -138,8 +138,10 @@ func GenerateJavascriptBundles() error { JavascriptBundles = make(map[string][]byte) JavascriptBundleChecksums = make(map[string]string) + jsMinifier := js.Minifier{Version: 2017} + minifier := minify.New() - minifier.AddFunc("text/javascript", js.Minify) + minifier.AddFunc("text/javascript", jsMinifier.Minify) for bundle, srcFiles := range bundles { var buffer bytes.Buffer From 9a637ce95e05459adc4712027e6a07eaabcfe657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Mon, 11 Mar 2024 20:43:14 -0700 Subject: [PATCH 6/8] Refactor RSS parser to use default namespace This change avoid some limitations of the Go XML parser regarding XML namespaces --- internal/reader/media/media.go | 1 + internal/reader/rss/atom.go | 43 ++++++ internal/reader/rss/parser.go | 4 +- internal/reader/rss/parser_test.go | 69 +++++----- internal/reader/rss/podcast.go | 42 ++++-- internal/reader/rss/rss.go | 207 +++++++++++------------------ 6 files changed, 185 insertions(+), 181 deletions(-) create mode 100644 internal/reader/rss/atom.go diff --git a/internal/reader/media/media.go b/internal/reader/media/media.go index 4d9c3661eda..df84bf03bce 100644 --- a/internal/reader/media/media.go +++ b/internal/reader/media/media.go @@ -12,6 +12,7 @@ import ( var textLinkRegex = regexp.MustCompile(`(?mi)(\bhttps?:\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])`) // Element represents XML media elements. +// Specs: https://www.rssboard.org/media-rss type Element struct { MediaGroups []Group `xml:"http://search.yahoo.com/mrss/ group"` MediaContents []Content `xml:"http://search.yahoo.com/mrss/ content"` diff --git a/internal/reader/rss/atom.go b/internal/reader/rss/atom.go new file mode 100644 index 00000000000..e0d66910d9d --- /dev/null +++ b/internal/reader/rss/atom.go @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package rss // import "miniflux.app/v2/internal/reader/rss" + +import "strings" + +type AtomAuthor struct { + Author AtomPerson `xml:"http://www.w3.org/2005/Atom author"` +} + +func (a *AtomAuthor) String() string { + return a.Author.String() +} + +type AtomPerson struct { + Name string `xml:"name"` + Email string `xml:"email"` +} + +func (a *AtomPerson) String() string { + var name string + + switch { + case a.Name != "": + name = a.Name + case a.Email != "": + name = a.Email + } + + return strings.TrimSpace(name) +} + +type AtomLink struct { + URL string `xml:"href,attr"` + Type string `xml:"type,attr"` + Rel string `xml:"rel,attr"` + Length string `xml:"length,attr"` +} + +type AtomLinks struct { + Links []*AtomLink `xml:"http://www.w3.org/2005/Atom link"` +} diff --git a/internal/reader/rss/parser.go b/internal/reader/rss/parser.go index a8390dc6b3a..55122ea4aad 100644 --- a/internal/reader/rss/parser.go +++ b/internal/reader/rss/parser.go @@ -14,7 +14,9 @@ import ( // Parse returns a normalized feed struct from a RSS feed. func Parse(baseURL string, data io.ReadSeeker) (*model.Feed, error) { feed := new(rssFeed) - if err := xml.NewXMLDecoder(data).Decode(feed); err != nil { + decoder := xml.NewXMLDecoder(data) + decoder.DefaultSpace = "rss" + if err := decoder.Decode(feed); err != nil { return nil, fmt.Errorf("rss: unable to parse feed: %w", err) } return feed.Transform(baseURL), nil diff --git a/internal/reader/rss/parser_test.go b/internal/reader/rss/parser_test.go index b3a46719b40..a8fbc76fd29 100644 --- a/internal/reader/rss/parser_test.go +++ b/internal/reader/rss/parser_test.go @@ -300,7 +300,7 @@ func TestParseEntryWithMultipleAtomLinks(t *testing.T) { Test - + ` @@ -430,7 +430,7 @@ func TestParseEntryWithAuthorAndCDATA(t *testing.T) { Test https://example.org/item - by + @@ -447,38 +447,6 @@ func TestParseEntryWithAuthorAndCDATA(t *testing.T) { } } -func TestParseEntryWithNonStandardAtomAuthor(t *testing.T) { - data := ` - - - Example - https://example.org/ - - - Test - https://example.org/item - - Foo Bar - Vice President - - FooBar Inc. - - - - ` - - feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) - if err != nil { - t.Fatal(err) - } - - expected := "Foo Bar" - result := feed.Entries[0].Author - if result != expected { - t.Errorf("Incorrect entry author, got %q instead of %q", result, expected) - } -} - func TestParseEntryWithAtomAuthorEmail(t *testing.T) { data := ` @@ -508,7 +476,7 @@ func TestParseEntryWithAtomAuthorEmail(t *testing.T) { } } -func TestParseEntryWithAtomAuthor(t *testing.T) { +func TestParseEntryWithAtomAuthorName(t *testing.T) { data := ` @@ -1435,6 +1403,37 @@ func TestEntryDescriptionFromGooglePlayDescription(t *testing.T) { } } +func TestParseEntryWithRSSDescriptionAndMediaDescription(t *testing.T) { + data := ` + + + Podcast Example + http://www.example.com/index.html + + Entry Title + http://www.example.com/entries/1 + Entry Description + Media Description + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries) != 1 { + t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries)) + } + + expected := "Entry Description" + result := feed.Entries[0].Content + if expected != result { + t.Errorf(`Unexpected description, got %q instead of %q`, result, expected) + } +} + func TestParseEntryWithCategoryAndInnerHTML(t *testing.T) { data := ` diff --git a/internal/reader/rss/podcast.go b/internal/reader/rss/podcast.go index b72426cc362..867bc03b3a3 100644 --- a/internal/reader/rss/podcast.go +++ b/internal/reader/rss/podcast.go @@ -15,21 +15,24 @@ var ErrInvalidDurationFormat = errors.New("rss: invalid duration format") // PodcastFeedElement represents iTunes and GooglePlay feed XML elements. // Specs: // - https://github.com/simplepie/simplepie-ng/wiki/Spec:-iTunes-Podcast-RSS -// - https://developers.google.com/search/reference/podcast/rss-feed +// - https://support.google.com/podcast-publishers/answer/9889544 type PodcastFeedElement struct { - ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd channel>author"` - Subtitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd channel>subtitle"` - Summary string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd channel>summary"` - PodcastOwner PodcastOwner `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd channel>owner"` - GooglePlayAuthor string `xml:"http://www.google.com/schemas/play-podcasts/1.0 channel>author"` + ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"` + Subtitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd subtitle"` + Summary string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd summary"` + PodcastOwner PodcastOwner `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd owner"` + GooglePlayAuthor string `xml:"http://www.google.com/schemas/play-podcasts/1.0 author"` } // PodcastEntryElement represents iTunes and GooglePlay entry XML elements. type PodcastEntryElement struct { - Subtitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd subtitle"` - Summary string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd summary"` - Duration string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd duration"` - GooglePlayDescription string `xml:"http://www.google.com/schemas/play-podcasts/1.0 description"` + ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"` + Subtitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd subtitle"` + Summary string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd summary"` + Duration string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd duration"` + PodcastOwner PodcastOwner `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd owner"` + GooglePlayAuthor string `xml:"http://www.google.com/schemas/play-podcasts/1.0 author"` + GooglePlayDescription string `xml:"http://www.google.com/schemas/play-podcasts/1.0 description"` } // PodcastOwner represents contact information for the podcast owner. @@ -38,6 +41,19 @@ type PodcastOwner struct { Email string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd email"` } +func (p *PodcastOwner) String() string { + var name string + + switch { + case p.Name != "": + name = p.Name + case p.Email != "": + name = p.Email + } + + return strings.TrimSpace(name) +} + // Image represents podcast artwork. type Image struct { URL string `xml:"href,attr"` @@ -52,10 +68,8 @@ func (e *PodcastFeedElement) PodcastAuthor() string { author = e.ItunesAuthor case e.GooglePlayAuthor != "": author = e.GooglePlayAuthor - case e.PodcastOwner.Name != "": - author = e.PodcastOwner.Name - case e.PodcastOwner.Email != "": - author = e.PodcastOwner.Email + case e.PodcastOwner.String() != "": + author = e.PodcastOwner.String() } return strings.TrimSpace(author) diff --git a/internal/reader/rss/rss.go b/internal/reader/rss/rss.go index 963b2d10716..cb769141d42 100644 --- a/internal/reader/rss/rss.go +++ b/internal/reader/rss/rss.go @@ -21,20 +21,25 @@ import ( "miniflux.app/v2/internal/urllib" ) -// Specs: https://cyber.harvard.edu/rss/rss.html +// Specs: https://www.rssboard.org/rss-specification type rssFeed struct { - XMLName xml.Name `xml:"rss"` - Version string `xml:"version,attr"` - Title string `xml:"channel>title"` - Links []rssLink `xml:"channel>link"` - ImageURL string `xml:"channel>image>url"` - Language string `xml:"channel>language"` - Description string `xml:"channel>description"` - PubDate string `xml:"channel>pubDate"` - ManagingEditor string `xml:"channel>managingEditor"` - Webmaster string `xml:"channel>webMaster"` - TimeToLive rssTTL `xml:"channel>ttl"` - Items []rssItem `xml:"channel>item"` + XMLName xml.Name `xml:"rss"` + Version string `xml:"rss version,attr"` + Channel rssChannel `xml:"rss channel"` +} + +type rssChannel struct { + Title string `xml:"rss title"` + Link string `xml:"rss link"` + ImageURL string `xml:"rss image>url"` + Language string `xml:"rss language"` + Description string `xml:"rss description"` + PubDate string `xml:"rss pubDate"` + ManagingEditor string `xml:"rss managingEditor"` + Webmaster string `xml:"rss webMaster"` + TimeToLive rssTTL `xml:"rss ttl"` + Items []rssItem `xml:"rss item"` + AtomLinks PodcastFeedElement } @@ -72,15 +77,15 @@ func (r *rssFeed) Transform(baseURL string) *model.Feed { feed.FeedURL = feedURL } - feed.Title = html.UnescapeString(strings.TrimSpace(r.Title)) + feed.Title = html.UnescapeString(strings.TrimSpace(r.Channel.Title)) if feed.Title == "" { feed.Title = feed.SiteURL } - feed.IconURL = strings.TrimSpace(r.ImageURL) - feed.TTL = r.TimeToLive.Value() + feed.IconURL = strings.TrimSpace(r.Channel.ImageURL) + feed.TTL = r.Channel.TimeToLive.Value() - for _, item := range r.Items { + for _, item := range r.Channel.Items { entry := item.Transform() if entry.Author == "" { entry.Author = r.feedAuthor() @@ -110,32 +115,29 @@ func (r *rssFeed) Transform(baseURL string) *model.Feed { } func (r *rssFeed) siteURL() string { - for _, element := range r.Links { - if element.XMLName.Space == "" { - return strings.TrimSpace(element.Data) - } - } - - return "" + return strings.TrimSpace(r.Channel.Link) } func (r *rssFeed) feedURL() string { - for _, element := range r.Links { - if element.XMLName.Space == "http://www.w3.org/2005/Atom" { - return strings.TrimSpace(element.Href) + for _, atomLink := range r.Channel.AtomLinks.Links { + if atomLink.Rel == "self" { + return strings.TrimSpace(atomLink.URL) } } - return "" } func (r rssFeed) feedAuthor() string { - author := r.PodcastAuthor() + author := r.Channel.PodcastAuthor() switch { - case r.ManagingEditor != "": - author = r.ManagingEditor - case r.Webmaster != "": - author = r.Webmaster + case r.Channel.ManagingEditor != "": + author = r.Channel.ManagingEditor + case r.Channel.Webmaster != "": + author = r.Channel.Webmaster + case r.Channel.GooglePlayAuthor != "": + author = r.Channel.GooglePlayAuthor + case r.Channel.PodcastOwner.String() != "": + author = r.Channel.PodcastOwner.String() } return sanitizer.StripTags(strings.TrimSpace(author)) } @@ -146,27 +148,7 @@ type rssGUID struct { IsPermaLink string `xml:"isPermaLink,attr"` } -type rssLink struct { - XMLName xml.Name - Data string `xml:",chardata"` - Href string `xml:"href,attr"` - Rel string `xml:"rel,attr"` -} - -type rssCommentLink struct { - XMLName xml.Name - Data string `xml:",chardata"` -} - type rssAuthor struct { - XMLName xml.Name - Data string `xml:",chardata"` - Name string `xml:"name"` - Email string `xml:"email"` - Inner string `xml:",innerxml"` -} - -type rssTitle struct { XMLName xml.Name Data string `xml:",chardata"` Inner string `xml:",innerxml"` @@ -193,19 +175,21 @@ func (enclosure *rssEnclosure) Size() int64 { } type rssItem struct { - GUID rssGUID `xml:"guid"` - Title []rssTitle `xml:"title"` - Links []rssLink `xml:"link"` - Description string `xml:"description"` - PubDate string `xml:"pubDate"` - Authors []rssAuthor `xml:"author"` - CommentLinks []rssCommentLink `xml:"comments"` - EnclosureLinks []rssEnclosure `xml:"enclosure"` - Categories []rssCategory `xml:"category"` + GUID rssGUID `xml:"rss guid"` + Title string `xml:"rss title"` + Link string `xml:"rss link"` + Description string `xml:"rss description"` + PubDate string `xml:"rss pubDate"` + Author rssAuthor `xml:"rss author"` + Comments string `xml:"rss comments"` + EnclosureLinks []rssEnclosure `xml:"rss enclosure"` + Categories []rssCategory `xml:"rss category"` dublincore.DublinCoreItemElement FeedBurnerElement PodcastEntryElement media.Element + AtomAuthor + AtomLinks } func (r *rssItem) Transform() *model.Entry { @@ -250,34 +234,26 @@ func (r *rssItem) entryDate() time.Time { } func (r *rssItem) entryAuthor() string { - author := "" - - for _, rssAuthor := range r.Authors { - switch rssAuthor.XMLName.Space { - case "http://www.itunes.com/dtds/podcast-1.0.dtd", "http://www.google.com/schemas/play-podcasts/1.0": - author = rssAuthor.Data - case "http://www.w3.org/2005/Atom": - if rssAuthor.Name != "" { - author = rssAuthor.Name - } else if rssAuthor.Email != "" { - author = rssAuthor.Email - } - default: - if rssAuthor.Name != "" { - author = rssAuthor.Name - } else if strings.Contains(rssAuthor.Inner, " Date: Mon, 11 Mar 2024 21:43:27 -0700 Subject: [PATCH 7/8] Move iTunes and GooglePlay XML definitions to their own packages --- internal/reader/googleplay/googleplay.go | 31 ++++++++++ internal/reader/itunes/itunes.go | 64 +++++++++++++++++++ internal/reader/rss/podcast.go | 78 ------------------------ internal/reader/rss/rss.go | 32 ++++++---- 4 files changed, 116 insertions(+), 89 deletions(-) create mode 100644 internal/reader/googleplay/googleplay.go create mode 100644 internal/reader/itunes/itunes.go diff --git a/internal/reader/googleplay/googleplay.go b/internal/reader/googleplay/googleplay.go new file mode 100644 index 00000000000..38dcc71fd19 --- /dev/null +++ b/internal/reader/googleplay/googleplay.go @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package googleplay // import "miniflux.app/v2/internal/reader/googleplay" + +// Specs: +// https://support.google.com/googleplay/podcasts/answer/6260341 +// https://www.google.com/schemas/play-podcasts/1.0/play-podcasts.xsd +type GooglePlayFeedElement struct { + GooglePlayAuthor string `xml:"http://www.google.com/schemas/play-podcasts/1.0 author"` + GooglePlayEmail string `xml:"http://www.google.com/schemas/play-podcasts/1.0 email"` + GooglePlayImage GooglePlayImageElement `xml:"http://www.google.com/schemas/play-podcasts/1.0 image"` + GooglePlayDescription string `xml:"http://www.google.com/schemas/play-podcasts/1.0 description"` + GooglePlayCategory GooglePlayCategoryElement `xml:"http://www.google.com/schemas/play-podcasts/1.0 category"` +} + +type GooglePlayItemElement struct { + GooglePlayAuthor string `xml:"http://www.google.com/schemas/play-podcasts/1.0 author"` + GooglePlayDescription string `xml:"http://www.google.com/schemas/play-podcasts/1.0 description"` + GooglePlayExplicit string `xml:"http://www.google.com/schemas/play-podcasts/1.0 explicit"` + GooglePlayBlock string `xml:"http://www.google.com/schemas/play-podcasts/1.0 block"` + GooglePlayNewFeedURL string `xml:"http://www.google.com/schemas/play-podcasts/1.0 new-feed-url"` +} + +type GooglePlayImageElement struct { + Href string `xml:"href,attr"` +} + +type GooglePlayCategoryElement struct { + Text string `xml:"text,attr"` +} diff --git a/internal/reader/itunes/itunes.go b/internal/reader/itunes/itunes.go new file mode 100644 index 00000000000..0382493ff21 --- /dev/null +++ b/internal/reader/itunes/itunes.go @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package itunes // import "miniflux.app/v2/internal/reader/itunes" + +import "strings" + +// Specs: https://help.apple.com/itc/podcasts_connect/#/itcb54353390 +type ItunesFeedElement struct { + ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"` + ItunesBlock string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd block"` + ItunesCategories []ItunesCategoryElement `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd category"` + ItunesComplete string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd complete"` + ItunesCopyright string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd copyright"` + ItunesExplicit string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd explicit"` + ItunesImage ItunesImageElement `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd image"` + Keywords string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd keywords"` + ItunesNewFeedURL string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd new-feed-url"` + ItunesOwner ItunesOwnerElement `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd owner"` + ItunesSummary string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd summary"` + ItunesTitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd title"` + ItunesType string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd type"` +} + +type ItunesItemElement struct { + ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"` + ItunesEpisode string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd episode"` + ItunesEpisodeType string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd episodeType"` + ItunesExplicit string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd explicit"` + ItunesDuration string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd duration"` + ItunesImage ItunesImageElement `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd image"` + ItunesSeason string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd season"` + ItunesSubtitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd subtitle"` + ItunesSummary string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd summary"` + ItunesTitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd title"` + ItunesTranscript string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd transcript"` +} + +type ItunesImageElement struct { + Href string `xml:"href,attr"` +} + +type ItunesCategoryElement struct { + Text string `xml:"text,attr"` + SubCategory *ItunesCategoryElement `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd category"` +} + +type ItunesOwnerElement struct { + Name string `xml:"name"` + Email string `xml:"email"` +} + +func (i *ItunesOwnerElement) String() string { + var name string + + switch { + case i.Name != "": + name = i.Name + case i.Email != "": + name = i.Email + } + + return strings.TrimSpace(name) +} diff --git a/internal/reader/rss/podcast.go b/internal/reader/rss/podcast.go index 867bc03b3a3..9a1f365bf2a 100644 --- a/internal/reader/rss/podcast.go +++ b/internal/reader/rss/podcast.go @@ -12,84 +12,6 @@ import ( var ErrInvalidDurationFormat = errors.New("rss: invalid duration format") -// PodcastFeedElement represents iTunes and GooglePlay feed XML elements. -// Specs: -// - https://github.com/simplepie/simplepie-ng/wiki/Spec:-iTunes-Podcast-RSS -// - https://support.google.com/podcast-publishers/answer/9889544 -type PodcastFeedElement struct { - ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"` - Subtitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd subtitle"` - Summary string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd summary"` - PodcastOwner PodcastOwner `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd owner"` - GooglePlayAuthor string `xml:"http://www.google.com/schemas/play-podcasts/1.0 author"` -} - -// PodcastEntryElement represents iTunes and GooglePlay entry XML elements. -type PodcastEntryElement struct { - ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"` - Subtitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd subtitle"` - Summary string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd summary"` - Duration string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd duration"` - PodcastOwner PodcastOwner `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd owner"` - GooglePlayAuthor string `xml:"http://www.google.com/schemas/play-podcasts/1.0 author"` - GooglePlayDescription string `xml:"http://www.google.com/schemas/play-podcasts/1.0 description"` -} - -// PodcastOwner represents contact information for the podcast owner. -type PodcastOwner struct { - Name string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd name"` - Email string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd email"` -} - -func (p *PodcastOwner) String() string { - var name string - - switch { - case p.Name != "": - name = p.Name - case p.Email != "": - name = p.Email - } - - return strings.TrimSpace(name) -} - -// Image represents podcast artwork. -type Image struct { - URL string `xml:"href,attr"` -} - -// PodcastAuthor returns the author of the podcast. -func (e *PodcastFeedElement) PodcastAuthor() string { - author := "" - - switch { - case e.ItunesAuthor != "": - author = e.ItunesAuthor - case e.GooglePlayAuthor != "": - author = e.GooglePlayAuthor - case e.PodcastOwner.String() != "": - author = e.PodcastOwner.String() - } - - return strings.TrimSpace(author) -} - -// PodcastDescription returns the description of the podcast. -func (e *PodcastEntryElement) PodcastDescription() string { - description := "" - - switch { - case e.GooglePlayDescription != "": - description = e.GooglePlayDescription - case e.Summary != "": - description = e.Summary - case e.Subtitle != "": - description = e.Subtitle - } - return strings.TrimSpace(description) -} - // normalizeDuration returns the duration tag value as a number of minutes func normalizeDuration(rawDuration string) (int, error) { var sumSeconds int diff --git a/internal/reader/rss/rss.go b/internal/reader/rss/rss.go index cb769141d42..cd1442bd4b5 100644 --- a/internal/reader/rss/rss.go +++ b/internal/reader/rss/rss.go @@ -16,6 +16,8 @@ import ( "miniflux.app/v2/internal/model" "miniflux.app/v2/internal/reader/date" "miniflux.app/v2/internal/reader/dublincore" + "miniflux.app/v2/internal/reader/googleplay" + "miniflux.app/v2/internal/reader/itunes" "miniflux.app/v2/internal/reader/media" "miniflux.app/v2/internal/reader/sanitizer" "miniflux.app/v2/internal/urllib" @@ -40,7 +42,8 @@ type rssChannel struct { TimeToLive rssTTL `xml:"rss ttl"` Items []rssItem `xml:"rss item"` AtomLinks - PodcastFeedElement + itunes.ItunesFeedElement + googleplay.GooglePlayFeedElement } type rssTTL struct { @@ -128,16 +131,18 @@ func (r *rssFeed) feedURL() string { } func (r rssFeed) feedAuthor() string { - author := r.Channel.PodcastAuthor() + var author string switch { + case r.Channel.ItunesAuthor != "": + author = r.Channel.ItunesAuthor + case r.Channel.GooglePlayAuthor != "": + author = r.Channel.GooglePlayAuthor + case r.Channel.ItunesOwner.String() != "": + author = r.Channel.ItunesOwner.String() case r.Channel.ManagingEditor != "": author = r.Channel.ManagingEditor case r.Channel.Webmaster != "": author = r.Channel.Webmaster - case r.Channel.GooglePlayAuthor != "": - author = r.Channel.GooglePlayAuthor - case r.Channel.PodcastOwner.String() != "": - author = r.Channel.PodcastOwner.String() } return sanitizer.StripTags(strings.TrimSpace(author)) } @@ -186,10 +191,11 @@ type rssItem struct { Categories []rssCategory `xml:"rss category"` dublincore.DublinCoreItemElement FeedBurnerElement - PodcastEntryElement media.Element AtomAuthor AtomLinks + itunes.ItunesItemElement + googleplay.GooglePlayItemElement } func (r *rssItem) Transform() *model.Entry { @@ -203,7 +209,7 @@ func (r *rssItem) Transform() *model.Entry { entry.Title = r.entryTitle() entry.Enclosures = r.entryEnclosures() entry.Tags = r.entryCategories() - if duration, err := normalizeDuration(r.Duration); err == nil { + if duration, err := normalizeDuration(r.ItunesDuration); err == nil { entry.ReadingTime = duration } @@ -237,8 +243,6 @@ func (r *rssItem) entryAuthor() string { var author string switch { - case r.PodcastOwner.String() != "": - author = r.PodcastOwner.String() case r.GooglePlayAuthor != "": author = r.GooglePlayAuthor case r.ItunesAuthor != "": @@ -277,7 +281,13 @@ func (r *rssItem) entryTitle() string { } func (r *rssItem) entryContent() string { - for _, value := range []string{r.DublinCoreContent, r.Description, r.PodcastDescription()} { + for _, value := range []string{ + r.DublinCoreContent, + r.Description, + r.GooglePlayDescription, + r.ItunesSummary, + r.ItunesSubtitle, + } { if value != "" { return value } From 6d97f8b4582414b6ce69467656824690057d4793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Mon, 11 Mar 2024 22:10:47 -0700 Subject: [PATCH 8/8] Parse podcast categories --- internal/reader/itunes/itunes.go | 11 +++ internal/reader/rss/parser_test.go | 110 ++++++++++++++++++++++++----- internal/reader/rss/rss.go | 32 +++------ 3 files changed, 113 insertions(+), 40 deletions(-) diff --git a/internal/reader/itunes/itunes.go b/internal/reader/itunes/itunes.go index 0382493ff21..1673f306bbd 100644 --- a/internal/reader/itunes/itunes.go +++ b/internal/reader/itunes/itunes.go @@ -22,6 +22,17 @@ type ItunesFeedElement struct { ItunesType string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd type"` } +func (i *ItunesFeedElement) GetItunesCategories() []string { + var categories []string + for _, category := range i.ItunesCategories { + categories = append(categories, category.Text) + if category.SubCategory != nil { + categories = append(categories, category.SubCategory.Text) + } + } + return categories +} + type ItunesItemElement struct { ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"` ItunesEpisode string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd episode"` diff --git a/internal/reader/rss/parser_test.go b/internal/reader/rss/parser_test.go index a8fbc76fd29..e4ff09edbb1 100644 --- a/internal/reader/rss/parser_test.go +++ b/internal/reader/rss/parser_test.go @@ -1434,18 +1434,17 @@ func TestParseEntryWithRSSDescriptionAndMediaDescription(t *testing.T) { } } -func TestParseEntryWithCategoryAndInnerHTML(t *testing.T) { +func TestParseFeedWithCategories(t *testing.T) { data := ` Example https://example.org/ - + Category 1 + Test https://example.org/item - Category 1 - Category 2 ` @@ -1459,27 +1458,99 @@ func TestParseEntryWithCategoryAndInnerHTML(t *testing.T) { t.Errorf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags)) } - expected := "Category 2" - result := feed.Entries[0].Tags[1] - if result != expected { - t.Errorf("Incorrect entry category, got %q instead of %q", result, expected) + expected := []string{"Category 1", "Category 2"} + result := feed.Entries[0].Tags + + for i, tag := range result { + if tag != expected[i] { + t.Errorf("Incorrect tag, got: %q", tag) + } } } -func TestParseEntryWithCategoryAndCDATA(t *testing.T) { +func TestParseEntryWithCategories(t *testing.T) { data := ` Example https://example.org/ - + Category 3 + + Test + https://example.org/item + Category 1 + + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries[0].Tags) != 3 { + t.Errorf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags)) + } + + expected := []string{"Category 1", "Category 2", "Category 3"} + result := feed.Entries[0].Tags + + for i, tag := range result { + if tag != expected[i] { + t.Errorf("Incorrect tag, got: %q", tag) + } + } +} + +func TestParseFeedWithItunesCategories(t *testing.T) { + data := ` + + + Example + https://example.org/ + + + + + + + + Test + https://example.org/item + + + ` + + feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data))) + if err != nil { + t.Fatal(err) + } + + if len(feed.Entries[0].Tags) != 4 { + t.Errorf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags)) + } + + expected := []string{"Society & Culture", "Documentary", "Health", "Mental Health"} + result := feed.Entries[0].Tags + + for i, tag := range result { + if tag != expected[i] { + t.Errorf("Incorrect tag, got: %q", tag) + } + } +} + +func TestParseFeedWithGooglePlayCategory(t *testing.T) { + data := ` + + + Example + https://example.org/ + Test https://example.org/item - - by - - Sample Category ` @@ -1493,10 +1564,13 @@ func TestParseEntryWithCategoryAndCDATA(t *testing.T) { t.Errorf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags)) } - expected := "Sample Category" - result := feed.Entries[0].Tags[0] - if result != expected { - t.Errorf("Incorrect entry category, got %q instead of %q", result, expected) + expected := []string{"Art"} + result := feed.Entries[0].Tags + + for i, tag := range result { + if tag != expected[i] { + t.Errorf("Incorrect tag, got: %q", tag) + } } } diff --git a/internal/reader/rss/rss.go b/internal/reader/rss/rss.go index cd1442bd4b5..be53c4b0d43 100644 --- a/internal/reader/rss/rss.go +++ b/internal/reader/rss/rss.go @@ -31,6 +31,7 @@ type rssFeed struct { } type rssChannel struct { + Categories []string `xml:"rss category"` Title string `xml:"rss title"` Link string `xml:"rss link"` ImageURL string `xml:"rss image>url"` @@ -111,6 +112,13 @@ func (r *rssFeed) Transform(baseURL string) *model.Feed { entry.Title = entry.URL } + entry.Tags = append(entry.Tags, r.Channel.Categories...) + entry.Tags = append(entry.Tags, r.Channel.GetItunesCategories()...) + + if r.Channel.GooglePlayCategory.Text != "" { + entry.Tags = append(entry.Tags, r.Channel.GooglePlayCategory.Text) + } + feed.Entries = append(feed.Entries, entry) } @@ -165,12 +173,6 @@ type rssEnclosure struct { Length string `xml:"length,attr"` } -type rssCategory struct { - XMLName xml.Name - Data string `xml:",chardata"` - Inner string `xml:",innerxml"` -} - func (enclosure *rssEnclosure) Size() int64 { if enclosure.Length == "" { return 0 @@ -188,7 +190,7 @@ type rssItem struct { Author rssAuthor `xml:"rss author"` Comments string `xml:"rss comments"` EnclosureLinks []rssEnclosure `xml:"rss enclosure"` - Categories []rssCategory `xml:"rss category"` + Categories []string `xml:"rss category"` dublincore.DublinCoreItemElement FeedBurnerElement media.Element @@ -208,7 +210,7 @@ func (r *rssItem) Transform() *model.Entry { entry.Content = r.entryContent() entry.Title = r.entryTitle() entry.Enclosures = r.entryEnclosures() - entry.Tags = r.entryCategories() + entry.Tags = r.Categories if duration, err := normalizeDuration(r.ItunesDuration); err == nil { entry.ReadingTime = duration } @@ -383,20 +385,6 @@ func (r *rssItem) entryEnclosures() model.EnclosureList { return enclosures } -func (r *rssItem) entryCategories() []string { - categoryList := make([]string, 0) - - for _, rssCategory := range r.Categories { - if strings.Contains(rssCategory.Inner, "