From 0b8120f5cc2ecbc5f366cb668d2309e729ddca78 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Mon, 19 Feb 2024 15:28:36 -0600 Subject: [PATCH 1/2] add an option to cache by album --- src/sigal/plugins/extended_caching.py | 97 ++++++++++++++++------- src/sigal/templates/sigal.conf.py | 5 ++ tests/test_extended_caching.py | 110 ++++++++++++++++++++++---- 3 files changed, 169 insertions(+), 43 deletions(-) diff --git a/src/sigal/plugins/extended_caching.py b/src/sigal/plugins/extended_caching.py index 4e0d8acd..64dcca5e 100644 --- a/src/sigal/plugins/extended_caching.py +++ b/src/sigal/plugins/extended_caching.py @@ -38,14 +38,35 @@ logger = logging.getLogger(__name__) +def _cache_key_global(path, name): + """Global (gallery) cache key function""" + return os.path.join(path, name) + + +def _cache_key_local(_, name): + """Local (album) cache key function""" + return name + + def load_metadata(album): """Loads the metadata of all media in an album from cache""" - if not hasattr(album.gallery, "metadataCache"): - _restore_cache(album.gallery) - cache = album.gallery.metadataCache + plugin_settings = album.gallery.settings.get("extended_caching_options", {}) + if plugin_settings.get("global_cache", True): + if not hasattr(album.gallery, "metadata_cache"): + logger.debug("Loading from global gallery cache") + cache_path = os.path.join(album.gallery.settings["destination"], ".metadata_cache") + _restore_cache(cache_path, album.gallery) + cache = album.gallery.metadata_cache + cache_key = _cache_key_global + else: + if not hasattr(album, "metadata_cache"): + logger.debug("Loading from local album cache %s", album.name) + _restore_cache(os.path.join(album.dst_path, ".metadata_cache"), album) + cache = album.metadata_cache + cache_key = _cache_key_local # load album metadata - key = os.path.join(album.path, "_index") + key = cache_key(album.path, "_index") if key in cache: data = cache[key] @@ -62,7 +83,7 @@ def load_metadata(album): # load media metadata for media in album.medias: - key = os.path.join(media.path, media.dst_filename) + key = cache_key(media.path, media.dst_filename) if key in cache: data = cache[key] @@ -92,36 +113,47 @@ def load_metadata(album): media.markdown_metadata = data["markdown_metadata"] -def _restore_cache(gallery): +def _restore_cache(cache_path, cache_object): """Restores the metadata cache from the cache file""" - cachePath = os.path.join(gallery.settings["destination"], ".metadata_cache") try: - if os.path.exists(cachePath): - with open(cachePath, "rb") as cacheFile: - gallery.metadataCache = pickle.load(cacheFile) - logger.debug("Loaded cache with %d entries", len(gallery.metadataCache)) + if os.path.exists(cache_path): + with open(cache_path, "rb") as cache_file: + cache_object.metadata_cache = pickle.load(cache_file) + logger.debug("Loaded cache with %d entries", len(cache_object.metadata_cache)) else: - gallery.metadataCache = {} + cache_object.metadata_cache = {} except Exception as e: logger.warning("Could not load cache: %s", e) - gallery.metadataCache = {} + cache_object.metadata_cache = {} -def save_cache(gallery): +def store_metadata(gallery): """Stores the exif data of all images in the gallery""" - - if hasattr(gallery, "metadataCache"): - cache = gallery.metadataCache + plugin_settings = gallery.settings.get("extended_caching_options", {}) + global_cache = plugin_settings.get("global_cache", True) + if global_cache: + logger.debug("Using global gallery cache") + if not hasattr(gallery, "metadata_cache"): + gallery.metadata_cache = {} + cache_key = _cache_key_global else: - cache = gallery.metadataCache = {} + logger.debug("Using local album caches") + cache_key = _cache_key_local for album in gallery.albums.values(): + if global_cache: + cache = gallery.metadata_cache + else: + if not hasattr(album, "metadata_cache"): + album.metadata_cache = {} + cache = album.metadata_cache + try: data = { "mod_date": int(get_mod_date(album.markdown_metadata_filepath)), "markdown_metadata": album.markdown_metadata, } - cache[os.path.join(album.path, "_index")] = data + cache[cache_key(album.path, "_index")] = data except FileNotFoundError: pass @@ -147,24 +179,33 @@ def save_cache(gallery): data["meta_mod_date"] = meta_mod_date data["markdown_metadata"] = media.markdown_metadata - cache[os.path.join(media.path, media.dst_filename)] = data + cache[cache_key(media.path, media.dst_filename)] = data + + if not global_cache: + cache_path = os.path.join(album.dst_path, ".metadata_cache") + _save_cache(cache_path, cache) + + if global_cache: + cache_path = os.path.join(gallery.settings["destination"], ".metadata_cache") + _save_cache(cache_path, gallery.metadata_cache) - cachePath = os.path.join(gallery.settings["destination"], ".metadata_cache") +def _save_cache(cache_path, cache): + """Stores the metadata cache to the cache file""" if len(cache) == 0: - if os.path.exists(cachePath): - os.remove(cachePath) + if os.path.exists(cache_path): + os.remove(cache_path) return try: - with open(cachePath, "wb") as cacheFile: - pickle.dump(cache, cacheFile) - logger.debug("Stored cache with %d entries", len(gallery.metadataCache)) + with open(cache_path, "wb") as cache_file: + pickle.dump(cache, cache_file) + logger.debug("Stored cache with %d entries", len(cache)) except Exception as e: logger.warn("Could not store cache: %s", e) - os.remove(cachePath) + os.remove(cache_path) def register(settings): - signals.gallery_build.connect(save_cache) + signals.gallery_build.connect(store_metadata) signals.album_initialized.connect(load_metadata) diff --git a/src/sigal/templates/sigal.conf.py b/src/sigal/templates/sigal.conf.py index d09e3a0f..0386b193 100644 --- a/src/sigal/templates/sigal.conf.py +++ b/src/sigal/templates/sigal.conf.py @@ -307,6 +307,11 @@ # 'ask_password': False # } +# Settings for extended caching plugin +# extended_caching_options = { +# 'global_cache': True # a single gallery cache, vs caching per album +# } + # Settings for nonmedia_files plugin. # nonmedia_files_options = { # 'ext_as_thumb': True, diff --git a/tests/test_extended_caching.py b/tests/test_extended_caching.py index 4617600e..388549a6 100644 --- a/tests/test_extended_caching.py +++ b/tests/test_extended_caching.py @@ -7,10 +7,10 @@ CURRENT_DIR = os.path.dirname(__file__) -def test_save_cache(settings, tmpdir): +def test_store_metadata(settings, tmpdir): settings["destination"] = str(tmpdir) gal = Gallery(settings, ncpu=1) - extended_caching.save_cache(gal) + extended_caching.store_metadata(gal) cachePath = os.path.join(settings["destination"], ".metadata_cache") @@ -49,7 +49,7 @@ def test_save_cache(settings, tmpdir): # test if file disappears gal.albums["exifTest"].medias.append(Image("foooo.jpg", "exifTest", settings)) - extended_caching.save_cache(gal) + extended_caching.store_metadata(gal) with open(cachePath, "rb") as cacheFile: cache = pickle.load(cacheFile) assert "exifTest/foooo.jpg" not in cache @@ -59,17 +59,97 @@ def test_restore_cache(settings, tmpdir): settings["destination"] = str(tmpdir) gal1 = Gallery(settings, ncpu=1) gal2 = Gallery(settings, ncpu=1) - extended_caching.save_cache(gal1) - extended_caching._restore_cache(gal2) - assert gal1.metadataCache == gal2.metadataCache + extended_caching.store_metadata(gal1) + cachePath = os.path.join(settings["destination"], ".metadata_cache") + extended_caching._restore_cache(cachePath, gal2) + assert gal1.metadata_cache == gal2.metadata_cache + + # test bad cache + with open(cachePath, "w") as f: + f.write("bad pickle file") + + extended_caching._restore_cache(cachePath, gal2) + assert gal2.metadata_cache == {} + + +def test_store_metadata_local(settings, tmpdir): + settings["destination"] = str(tmpdir) + settings['extended_caching_options'] = {'global_cache': False} + gal = Gallery(settings, ncpu=1) + extended_caching.store_metadata(gal) + + for album in gal.albums.values(): + if album.metadata_cache: + cachePath = os.path.join(album.dst_path, ".metadata_cache") + assert os.path.isfile(cachePath) + with open(cachePath, "rb") as cacheFile: + cache = pickle.load(cacheFile) + + # test exif + cachePath = os.path.join(settings["destination"], "exifTest", ".metadata_cache") + assert os.path.isfile(cachePath) + with open(cachePath, "rb") as cacheFile: + cache = pickle.load(cacheFile) + + album = gal.albums["exifTest"] + cache_img = cache["21.jpg"] + assert cache_img["exif"] == album.medias[0].exif + assert "markdown_metadata" not in cache_img + assert cache_img["file_metadata"] == album.medias[0].file_metadata + + cache_img = cache["22.jpg"] + assert cache_img["exif"] == album.medias[1].exif + assert "markdown_metadata" not in cache_img + assert cache_img["file_metadata"] == album.medias[1].file_metadata + + cache_img = cache["noexif.png"] + assert cache_img["exif"] == album.medias[2].exif + assert "markdown_metadata" not in cache_img + assert cache_img["file_metadata"] == album.medias[2].file_metadata + + # test iptc and md + cachePath = os.path.join(settings["destination"], "iptcTest", ".metadata_cache") + assert os.path.isfile(cachePath) + with open(cachePath, "rb") as cacheFile: + cache = pickle.load(cacheFile) + + album = gal.albums["iptcTest"] + assert cache["_index"]["markdown_metadata"] == album.markdown_metadata + + cache_img = cache["1.jpg"] + assert cache_img["file_metadata"] == album.medias[0].file_metadata + assert "markdown_metadata" not in cache_img + + cache_img = cache["2.jpg"] + assert cache_img["markdown_metadata"] == album.medias[1].markdown_metadata + + # test if file disappears + gal.albums["exifTest"].medias.append(Image("foooo.jpg", "exifTest", settings)) + extended_caching.store_metadata(gal) + cachePath = os.path.join(settings["destination"], "exifTest", ".metadata_cache") + with open(cachePath, "rb") as cacheFile: + cache = pickle.load(cacheFile) + assert "foooo.jpg" not in cache + + +def test_restore_cache_local(settings, tmpdir): + settings["destination"] = str(tmpdir) + settings['extended_caching_options'] = {'global_cache': False} + gal1 = Gallery(settings, ncpu=1) + gal2 = Gallery(settings, ncpu=1) + extended_caching.store_metadata(gal1) + cachePath = os.path.join(settings["destination"], "exifTest", ".metadata_cache") + extended_caching._restore_cache(cachePath, gal2.albums["exifTest"]) + assert not hasattr(gal1, "metadata_cache") + assert not hasattr(gal2, "metadata_cache") + assert gal1.albums["exifTest"].metadata_cache == gal2.albums["exifTest"].metadata_cache # test bad cache - cachePath = os.path.join(settings["destination"], ".metadata_cache") with open(cachePath, "w") as f: f.write("bad pickle file") - extended_caching._restore_cache(gal2) - assert gal2.metadataCache == {} + extended_caching._restore_cache(cachePath, gal2.albums["exifTest"]) + assert gal2.albums["exifTest"].metadata_cache == {} def test_load_exif(settings, tmpdir): @@ -77,7 +157,7 @@ def test_load_exif(settings, tmpdir): gal1 = Gallery(settings, ncpu=1) gal1.albums["exifTest"].medias[2].exif = "blafoo" # set mod_date in future, to force these values - gal1.metadataCache = { + gal1.metadata_cache = { "exifTest/21.jpg": {"exif": "Foo", "mod_date": 100000000000}, "exifTest/22.jpg": {"exif": "Bar", "mod_date": 100000000000}, } @@ -88,9 +168,9 @@ def test_load_exif(settings, tmpdir): assert gal1.albums["exifTest"].medias[1].exif == "Bar" assert gal1.albums["exifTest"].medias[2].exif == "blafoo" - # check if setting gallery.metadataCache works + # check if setting gallery.metadata_cache works gal2 = Gallery(settings, ncpu=1) - extended_caching.save_cache(gal1) + extended_caching.store_metadata(gal1) extended_caching.load_metadata(gal2.albums["exifTest"]) assert gal2.albums["exifTest"].medias[0].exif == "Foo" @@ -101,14 +181,14 @@ def test_load_exif(settings, tmpdir): def test_load_metadata_missing(settings, tmpdir): settings["destination"] = str(tmpdir) gal = Gallery(settings, ncpu=1) - extended_caching.save_cache(gal) - assert gal.metadataCache + extended_caching.store_metadata(gal) + assert gal.metadata_cache # test if file disappears gal.albums["exifTest"].medias.append(Image("foooo.jpg", "exifTest", settings)) # set mod_date to -1 to force cache update - gal.metadataCache = { + gal.metadata_cache = { "exifTest/_index": { "mod_date": -1, }, From 2b2d6e04244cdb33f6d582fc072478cc4a83a6d5 Mon Sep 17 00:00:00 2001 From: David Schultz Date: Mon, 19 Feb 2024 21:57:30 -0600 Subject: [PATCH 2/2] more caching tests for better coverage --- src/sigal/plugins/extended_caching.py | 2 +- tests/test_extended_caching.py | 87 ++++++++++++++++++++------- 2 files changed, 65 insertions(+), 24 deletions(-) diff --git a/src/sigal/plugins/extended_caching.py b/src/sigal/plugins/extended_caching.py index 64dcca5e..c4da6a02 100644 --- a/src/sigal/plugins/extended_caching.py +++ b/src/sigal/plugins/extended_caching.py @@ -202,7 +202,7 @@ def _save_cache(cache_path, cache): pickle.dump(cache, cache_file) logger.debug("Stored cache with %d entries", len(cache)) except Exception as e: - logger.warn("Could not store cache: %s", e) + logger.warning("Could not store cache: %s", e) os.remove(cache_path) diff --git a/tests/test_extended_caching.py b/tests/test_extended_caching.py index 388549a6..3e3e5fc9 100644 --- a/tests/test_extended_caching.py +++ b/tests/test_extended_caching.py @@ -12,11 +12,11 @@ def test_store_metadata(settings, tmpdir): gal = Gallery(settings, ncpu=1) extended_caching.store_metadata(gal) - cachePath = os.path.join(settings["destination"], ".metadata_cache") + cache_path = os.path.join(settings["destination"], ".metadata_cache") - assert os.path.isfile(cachePath) + assert os.path.isfile(cache_path) - with open(cachePath, "rb") as cacheFile: + with open(cache_path, "rb") as cacheFile: cache = pickle.load(cacheFile) # test exif @@ -50,25 +50,36 @@ def test_store_metadata(settings, tmpdir): # test if file disappears gal.albums["exifTest"].medias.append(Image("foooo.jpg", "exifTest", settings)) extended_caching.store_metadata(gal) - with open(cachePath, "rb") as cacheFile: + with open(cache_path, "rb") as cacheFile: cache = pickle.load(cacheFile) assert "exifTest/foooo.jpg" not in cache +def test_load_metadata(settings, tmpdir): + settings["destination"] = str(tmpdir) + gal1 = Gallery(settings, ncpu=1) + gal2 = Gallery(settings, ncpu=1) + extended_caching.store_metadata(gal1) + for album in gal2.albums.values(): + extended_caching.load_metadata(album) + break # only need to load one + assert gal1.metadata_cache == gal2.metadata_cache + + def test_restore_cache(settings, tmpdir): settings["destination"] = str(tmpdir) gal1 = Gallery(settings, ncpu=1) gal2 = Gallery(settings, ncpu=1) extended_caching.store_metadata(gal1) - cachePath = os.path.join(settings["destination"], ".metadata_cache") - extended_caching._restore_cache(cachePath, gal2) + cache_path = os.path.join(settings["destination"], ".metadata_cache") + extended_caching._restore_cache(cache_path, gal2) assert gal1.metadata_cache == gal2.metadata_cache # test bad cache - with open(cachePath, "w") as f: + with open(cache_path, "w") as f: f.write("bad pickle file") - extended_caching._restore_cache(cachePath, gal2) + extended_caching._restore_cache(cache_path, gal2) assert gal2.metadata_cache == {} @@ -80,15 +91,15 @@ def test_store_metadata_local(settings, tmpdir): for album in gal.albums.values(): if album.metadata_cache: - cachePath = os.path.join(album.dst_path, ".metadata_cache") - assert os.path.isfile(cachePath) - with open(cachePath, "rb") as cacheFile: + cache_path = os.path.join(album.dst_path, ".metadata_cache") + assert os.path.isfile(cache_path) + with open(cache_path, "rb") as cacheFile: cache = pickle.load(cacheFile) # test exif - cachePath = os.path.join(settings["destination"], "exifTest", ".metadata_cache") - assert os.path.isfile(cachePath) - with open(cachePath, "rb") as cacheFile: + cache_path = os.path.join(settings["destination"], "exifTest", ".metadata_cache") + assert os.path.isfile(cache_path) + with open(cache_path, "rb") as cacheFile: cache = pickle.load(cacheFile) album = gal.albums["exifTest"] @@ -108,9 +119,9 @@ def test_store_metadata_local(settings, tmpdir): assert cache_img["file_metadata"] == album.medias[2].file_metadata # test iptc and md - cachePath = os.path.join(settings["destination"], "iptcTest", ".metadata_cache") - assert os.path.isfile(cachePath) - with open(cachePath, "rb") as cacheFile: + cache_path = os.path.join(settings["destination"], "iptcTest", ".metadata_cache") + assert os.path.isfile(cache_path) + with open(cache_path, "rb") as cacheFile: cache = pickle.load(cacheFile) album = gal.albums["iptcTest"] @@ -126,8 +137,8 @@ def test_store_metadata_local(settings, tmpdir): # test if file disappears gal.albums["exifTest"].medias.append(Image("foooo.jpg", "exifTest", settings)) extended_caching.store_metadata(gal) - cachePath = os.path.join(settings["destination"], "exifTest", ".metadata_cache") - with open(cachePath, "rb") as cacheFile: + cache_path = os.path.join(settings["destination"], "exifTest", ".metadata_cache") + with open(cache_path, "rb") as cacheFile: cache = pickle.load(cacheFile) assert "foooo.jpg" not in cache @@ -138,20 +149,34 @@ def test_restore_cache_local(settings, tmpdir): gal1 = Gallery(settings, ncpu=1) gal2 = Gallery(settings, ncpu=1) extended_caching.store_metadata(gal1) - cachePath = os.path.join(settings["destination"], "exifTest", ".metadata_cache") - extended_caching._restore_cache(cachePath, gal2.albums["exifTest"]) + cache_path = os.path.join(settings["destination"], "exifTest", ".metadata_cache") + extended_caching._restore_cache(cache_path, gal2.albums["exifTest"]) assert not hasattr(gal1, "metadata_cache") assert not hasattr(gal2, "metadata_cache") assert gal1.albums["exifTest"].metadata_cache == gal2.albums["exifTest"].metadata_cache # test bad cache - with open(cachePath, "w") as f: + with open(cache_path, "w") as f: f.write("bad pickle file") - extended_caching._restore_cache(cachePath, gal2.albums["exifTest"]) + extended_caching._restore_cache(cache_path, gal2.albums["exifTest"]) assert gal2.albums["exifTest"].metadata_cache == {} +def test_load_metadata_local(settings, tmpdir): + settings["destination"] = str(tmpdir) + settings['extended_caching_options'] = {'global_cache': False} + gal1 = Gallery(settings, ncpu=1) + gal2 = Gallery(settings, ncpu=1) + extended_caching.store_metadata(gal1) + for album in gal2.albums.values(): + extended_caching.load_metadata(album) + assert not hasattr(gal1, "metadata_cache") + assert not hasattr(gal2, "metadata_cache") + for al1, al2 in zip(gal1.albums.values(), gal2.albums.values()): + assert al1.metadata_cache == al2.metadata_cache + + def test_load_exif(settings, tmpdir): settings["destination"] = str(tmpdir) gal1 = Gallery(settings, ncpu=1) @@ -209,3 +234,19 @@ def test_load_metadata_missing(settings, tmpdir): extended_caching.load_metadata(gal.albums["dir1/test2"]) assert gal.albums["dir1/test2"].medias[1].exif == "Bar" assert gal.albums["dir1/test2"].medias[1].markdown_metadata != "Bar" + + +def test_empty_cache(settings, tmpdir): + cache_path = os.path.join(tmpdir, ".metadata_cache") + with open(cache_path, 'w') as f: + f.write("bad pickle file") + extended_caching._save_cache(cache_path, {}) + assert not os.path.exists(cache_path) + + +def test_uncachable(settings, tmpdir): + cache_path = os.path.join(tmpdir, ".metadata_cache") + with open(cache_path, 'w') as f: + f.write("bad pickle file") + extended_caching._save_cache(cache_path, {'func': lambda x: x+1}) + assert not os.path.exists(cache_path)