Add support for caching and reusing packages

aea/package_manager/base.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -86,6 +86,53 @@ class DepedencyMismatchErrors(Enum):
     HASH_DOES_NOT_MATCH: int = 2
 
 
+class Cache:
+    """Cache manager."""
+
+    def __init__(self) -> None:
+        """Package cache helper."""
+        self.path = Path.home() / ".aea" / "cache" / "packages"
+        self.path.mkdir(parents=True, exist_ok=True)
+
+    def exists(self, package_hash: str) -> bool:
+        """Check if package exists in the cache."""
+        return (self.path / package_hash).exists()
+
+    def add(self, package_path: Path, package_hash: str) -> None:
+        """Add package to cache."""
+        path = self.path / package_hash
+        if path.exists() and self.valid(package_hash=package_hash):
+            return
+
+        if path.exists():
+            shutil.rmtree(path=path)
+
+        path.mkdir()
+        shutil.copytree(package_path, path / package_path.name)
+
+    def remove(self, package_hash: str) -> None:
+        """Remove package."""
+        path = self.path / package_hash
+        if path.exists():
+            shutil.rmtree(path=path)
+
+    def valid(self, package_hash: str) -> bool:
+        """Validate a package."""
+        (package_path,) = (self.path / package_hash).iterdir()
+        cache_hash = IPFSHashOnly.hash_directory(dir_path=str(package_path))
+        return cache_hash == package_hash
+
+    def copy(self, package_hash: str, destination_path: Path) -> bool:
+        """Copy package from cache."""
+        if not self.valid(package_hash=package_hash):
+            self.remove(package_hash=package_hash)
+            return False
+
+        (package_path,) = (self.path / package_hash).iterdir()
+        shutil.copytree(package_path, destination_path)
+        return True
+
+
 class BasePackageManager(ABC):
     """AEA package manager"""
 
@@ -103,6 +150,8 @@ def __init__(
         self.config_loader = config_loader
         self._packages_file = path / PACKAGES_FILE
 
+        self.cache = Cache()
+
         self._logger = logger or logging.getLogger(name="PackageManager")
         self._logger.setLevel(logging.INFO)
 
@@ -368,12 +417,24 @@ def _fetch_package(self, package_id: PackageId) -> None:
             (package_type_collection / "__init__.py").touch()
 
         download_path = package_type_collection / package_id.name
-        load_fetch_ipfs()(
-            str(package_id.package_type),
-            package_id.public_id,
-            str(download_path),
-            True,
-        )
+        fetched = False
+        if self.cache.exists(package_id.package_hash):
+            fetched = self.cache.copy(
+                package_hash=package_id.package_hash,
+                destination_path=download_path,
+            )
+
+        if not fetched:
+            load_fetch_ipfs()(
+                str(package_id.package_type),
+                package_id.public_id,
+                str(download_path),
+                True,
+            )
+            self.cache.add(
+                package_path=download_path,
+                package_hash=package_id.package_hash,
+            )
         self._logger.debug(f"Downloaded {package_id.without_hash()}")
 
     def add_dependencies_for_package(

docs/api/package_manager/base.md

@@ -44,6 +44,76 @@ class DepedencyMismatchErrors(Enum)
 
 Dependency mismatch errors.
 
+<a id="aea.package_manager.base.Cache"></a>
+
+## Cache Objects
+
+```python
+class Cache()
+```
+
+Cache manager.
+
+<a id="aea.package_manager.base.Cache.__init__"></a>
+
+#### `__`init`__`
+
+```python
+def __init__() -> None
+```
+
+Package cache helper.
+
+<a id="aea.package_manager.base.Cache.exists"></a>
+
+#### exists
+
+```python
+def exists(package_hash: str) -> bool
+```
+
+Check if package exists in the cache.
+
+<a id="aea.package_manager.base.Cache.add"></a>
+
+#### add
+
+```python
+def add(package_path: Path, package_hash: str) -> None
+```
+
+Add package to cache.
+
+<a id="aea.package_manager.base.Cache.remove"></a>
+
+#### remove
+
+```python
+def remove(package_hash: str) -> None
+```
+
+Remove package.
+
+<a id="aea.package_manager.base.Cache.valid"></a>
+
+#### valid
+
+```python
+def valid(package_hash: str) -> bool
+```
+
+Validate a package.
+
+<a id="aea.package_manager.base.Cache.copy"></a>
+
+#### copy
+
+```python
+def copy(package_hash: str, destination_path: Path) -> bool
+```
+
+Copy package from cache.
+
 <a id="aea.package_manager.base.BasePackageManager"></a>
 
 ## BasePackageManager Objects

tests/test_package_manager/test_base.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -94,6 +94,7 @@ def __init__(
         self.path = path
         self.packages = packages
         self.config_loader = config_loader
+        self.cache = mock.Mock(exists=lambda *x: False)
         self._logger = logging.getLogger()
 
     @classmethod
@@ -190,9 +191,12 @@ def test_add_package(
         )
         dummy_package = PackageId(
             package_type=PackageType.SKILL,
-            public_id=PublicId(author="dummy", name="skill"),
+            public_id=PublicId(
+                author="dummy",
+                name="skill",
+                package_hash="bafybeicoawiackcbgqo3na3e56tpdc62atag4yxknur77py37caqq4mmya",
+            ),
         )
-
         with mock.patch(
             "aea.package_manager.base.load_fetch_ipfs", new=lambda: _fetch_ipfs_mock
         ):

tests/test_package_manager/test_v0.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -289,7 +289,8 @@ def test_missing_hash(self, caplog) -> None:
 
 
 @mock.patch("aea.package_manager.base.load_fetch_ipfs")
-def test_package_manager_add_item_dependency_support_mock(fetch_mock):
+@mock.patch("aea.package_manager.base.Cache")
+def test_package_manager_add_item_dependency_support_mock(*mocks):
     """Check PackageManager.add_packages works with dependencies on mocks."""
     FAKE_PACKAGES = [
         PackageId(

tests/test_package_manager/test_v1.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022-2023 Valory AG
+#   Copyright 2022-2024 Valory AG
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -708,7 +708,8 @@ def test_package_manager_add_item_dependency_support():
 
 
 @mock.patch("aea.package_manager.base.load_fetch_ipfs")
-def test_package_manager_add_item_dependency_support_mock(fetch_mock):
+@mock.patch("aea.package_manager.base.Cache")
+def test_package_manager_add_item_dependency_support_mock(*mocks):
     """Check PackageManager.add_packages works with dependencies on mocks."""
     FAKE_PACKAGES = [
         PackageId(
@@ -770,7 +771,8 @@ def test_package_manager_add_package_already_installed(fetch_mock: mock.Mock):
 
 
 @mock.patch("aea.package_manager.base.load_fetch_ipfs")
-def test_package_manager_add_package_can_be_updated(fetch_mock: mock.Mock):
+@mock.patch("aea.package_manager.base.Cache")
+def test_package_manager_add_package_can_be_updated(*mocks: mock.Mock):
     """Test package update on add_package."""
     # version already installed
     data = TEST_SKILL_ID.public_id.json
@@ -795,7 +797,7 @@ def test_package_manager_add_package_can_be_updated(fetch_mock: mock.Mock):
                 match="Required package and package in the registry does not match",
             ):
                 package_manager.add_package(TEST_SKILL_ID)
-            fetch_mock.assert_not_called()
+            mocks[1].assert_not_called()
 
             package_manager.add_package(TEST_SKILL_ID, allow_update=True)
             remove_mock.assert_called_once_with(TEST_SKILL_ID)