Support dependency overrides

.gitignore

@@ -154,4 +154,5 @@ coverage.txt
 
 temp/
 agent_name/
+agent/
 leak_report

HISTORY.md

@@ -1,6 +1,12 @@
 # Release History - open AEA
 
 
+## 1.40.0 (2023-09-26)
+
+AEA:
+- Adds support for specifying extra dependencies and overriding dependencies via `-e` flag on `aea install`
+- Updates the selection of dependencies in `aea install` command to override the dependencies in the `extra dependencies provided by flag > agent > skill > connection > contract > protocol` order instead of merging them.
+
 ## 1.40.0 (2023-09-26)
 
 AEA:

aea/cli/install.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022 Valory AG
+#   Copyright 2022-2023 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,15 +19,15 @@
 # ------------------------------------------------------------------------------
 """Implementation of the 'aea install' subcommand."""
 
-from typing import Optional, cast
+from typing import Optional, Tuple, cast
 
 import click
 
+from aea.cli.utils.click_utils import PyPiDependency
 from aea.cli.utils.context import Context
 from aea.cli.utils.decorators import check_aea_project
 from aea.cli.utils.loggers import logger
-from aea.configurations.data_types import Dependencies
-from aea.configurations.pypi import is_satisfiable, is_simple_dep, to_set_specifier
+from aea.configurations.data_types import Dependency
 from aea.exceptions import AEAException
 from aea.helpers.install_dependency import call_pip, install_dependencies
 
@@ -41,66 +41,56 @@
     default=None,
     help="Install from the given requirements file.",
 )
+@click.option(
+    "-e",
+    "--extra-dependency",
+    "extra_dependencies",
+    type=PyPiDependency(),
+    help="Provide extra dependency.",
+    multiple=True,
+)
 @click.pass_context
 @check_aea_project
-def install(click_context: click.Context, requirement: Optional[str]) -> None:
+def install(
+    click_context: click.Context,
+    requirement: Optional[str],
+    extra_dependencies: Tuple[Dependency],
+) -> None:
     """Install the dependencies of the agent."""
     ctx = cast(Context, click_context.obj)
-    do_install(ctx, requirement)
+    do_install(ctx, requirement, extra_dependencies)
 
 
-def do_install(ctx: Context, requirement: Optional[str] = None) -> None:
+def do_install(
+    ctx: Context,
+    requirement: Optional[str] = None,
+    extra_dependencies: Optional[Tuple[Dependency]] = None,
+) -> None:
     """
     Install necessary dependencies.
 
     :param ctx: context object.
     :param requirement: optional str requirement.
+    :param extra_dependencies: List of the extra dependencies to use
 
     :raises ClickException: if AEAException occurs.
     """
     try:
         if requirement:
+            if extra_dependencies is not None and len(extra_dependencies) > 0:
+                logger.debug(
+                    "Extra dependencies will be ignored while installing from requirements file"
+                )
             logger.debug("Installing the dependencies in '{}'...".format(requirement))
             _install_from_requirement(requirement)
         else:
             logger.debug("Installing all the dependencies...")
-            dependencies = ctx.get_dependencies()
-
-            logger.debug("Preliminary check on satisfiability of version specifiers...")
-            unsat_dependencies = _find_unsatisfiable_dependencies(dependencies)
-            if len(unsat_dependencies) != 0:
-                raise AEAException(
-                    "cannot install the following dependencies "
-                    + "as the joint version specifier is unsatisfiable:\n - "
-                    + "\n -".join(
-                        [
-                            f"{name}: {to_set_specifier(dep)}"
-                            for name, dep in unsat_dependencies.items()
-                        ]
-                    )
-                )
+            dependencies = ctx.get_dependencies(extra_dependencies=extra_dependencies)
             install_dependencies(list(dependencies.values()), logger=logger)
     except AEAException as e:
         raise click.ClickException(str(e))
 
 
-def _find_unsatisfiable_dependencies(dependencies: Dependencies) -> Dependencies:
-    """
-    Find unsatisfiable dependencies.
-
-    It only checks among 'simple' dependencies (i.e. if it has no field specified,
-    or only the 'version' field set.)
-
-    :param dependencies: the dependencies to check.
-    :return: the unsatisfiable dependencies.
-    """
-    return {
-        name: dep
-        for name, dep in dependencies.items()
-        if is_simple_dep(dep) and not is_satisfiable(to_set_specifier(dep))
-    }
-
-
 def _install_from_requirement(file: str, install_timeout: float = 300) -> None:
     """
     Install from requirements.

aea/cli/utils/click_utils.py

@@ -45,7 +45,7 @@
     PROTOCOL,
     SKILL,
 )
-from aea.configurations.data_types import PackageType, PublicId
+from aea.configurations.data_types import Dependency, PackageType, PublicId
 from aea.helpers.io import open_file
 from aea.package_manager.base import PACKAGE_SOURCE_RE
 
@@ -244,6 +244,21 @@ def convert(self, value: str, param: Any, ctx: click.Context) -> str:
         return value
 
 
+class PyPiDependency(click.ParamType):
+    """Click parameter for PyPy dependency string"""
+
+    def get_metavar(self, param: Any) -> str:
+        """Return the metavar default for this param if it provides one."""
+        return "DEPENDENCY"  # pragma: no cover
+
+    def convert(self, value: str, param: Any, ctx: click.Context) -> Dependency:
+        """Convert the value."""
+        try:
+            return Dependency.from_string(value)
+        except ValueError as e:
+            raise click.ClickException(str(e)) from e
+
+
 def registry_flag(
     mark_default: bool = True,
     default_registry: Optional[str] = None,

aea/cli/utils/context.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022 Valory AG
+#   Copyright 2022-2023 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,28 +20,31 @@
 """A module with context tools of the aea cli."""
 import os
 from pathlib import Path
-from typing import Any, Dict, List, Optional, cast
+from typing import Any, Dict, List, Optional, Tuple, cast
 
 from aea.cli.registry.settings import REGISTRY_LOCAL, REGISTRY_TYPES
 from aea.cli.utils.loggers import logger
 from aea.configurations.base import (
     AgentConfig,
     Dependencies,
+    Dependency,
     PackageType,
     PublicId,
     _get_default_configuration_file_name_from_type,
 )
 from aea.configurations.constants import (
-    CONNECTION,
-    CONTRACT,
     DEFAULT_AEA_CONFIG_FILE,
     DEFAULT_REGISTRY_NAME,
-    PROTOCOL,
-    SKILL,
     VENDOR,
 )
 from aea.configurations.loader import ConfigLoader
-from aea.configurations.pypi import merge_dependencies_list
+from aea.configurations.pypi import (
+    is_satisfiable,
+    is_simple_dep,
+    merge_dependencies_list,
+    to_set_specifier,
+)
+from aea.exceptions import AEAException
 from aea.helpers.io import open_file
 
 
@@ -184,39 +187,96 @@ def _get_item_dependencies(item_type: str, public_id: PublicId) -> Dependencies:
         deps = cast(Dependencies, config.dependencies)
         return deps
 
-    def get_dependencies(self) -> Dependencies:
+    @staticmethod
+    def _find_unsatisfiable_dependencies(dependencies: Dependencies) -> Dependencies:
+        """
+        Find unsatisfiable dependencies.
+
+        It only checks among 'simple' dependencies (i.e. if it has no field specified,
+        or only the 'version' field set.)
+
+        :param dependencies: the dependencies to check.
+        :return: the unsatisfiable dependencies.
+        """
+        return {
+            name: dep
+            for name, dep in dependencies.items()
+            if is_simple_dep(dep) and not is_satisfiable(to_set_specifier(dep))
+        }
+
+    def _get_dependencies_by_item_type(self, item_type: PackageType) -> Dependencies:
+        """Get the dependencies from item type and public id."""
+        if item_type == PackageType.AGENT:
+            return self.agent_config.dependencies
+        dependency_to_package: Dict[str, List[Tuple[PublicId, Dependency]]] = {}
+        dependencies = []
+        for item_id in getattr(self.agent_config, item_type.to_plural()):
+            package_dependencies = self._get_item_dependencies(item_type.value, item_id)
+            dependencies += [package_dependencies]
+            for dep, spec in package_dependencies.items():
+                if dep not in dependency_to_package:
+                    dependency_to_package[dep] = []
+                dependency_to_package[dep].append((item_id, spec))
+
+        merged_dependencies = merge_dependencies_list(*dependencies)
+        unsat_dependencies = self._find_unsatisfiable_dependencies(merged_dependencies)
+        if len(unsat_dependencies) > 0:
+            error = f"Error while merging dependencies for {item_type.to_plural()}"
+            error += "; Joint version specifier is unsatisfiable for following dependencies:\n"
+            error += "======================================\n"
+            for name, spec in unsat_dependencies.items():
+                error += f"Dependency: {name}\n"
+                error += f"Specifier: {to_set_specifier(spec)}\n"
+                error += "Packages containing dependency: \n"
+                for package, dep_spec in dependency_to_package[name]:
+                    error += f"  - {package.without_hash()}: {dep_spec.get_pip_install_args()[0]}\n"
+                error += "======================================\n"
+
+            raise AEAException(error[:-1])
+        return merged_dependencies
+
+    def get_dependencies(
+        self,
+        extra_dependencies: Optional[Tuple[Dependency]] = None,
+    ) -> Dependencies:
         """
         Aggregate the dependencies from every component.
 
+        :param extra_dependencies: List of the extra dependencies to use, if the
+                                extra dependencies and agent dependencies have conflicts
+                                the packages from extra dependencies list will be prefered
+                                over the agent dependencies
         :return: a list of dependency version specification. e.g. ["gym >= 1.0.0"]
         """
-        protocol_dependencies = [
-            self._get_item_dependencies(PROTOCOL, protocol_id)
-            for protocol_id in self.agent_config.protocols
-        ]
-        connection_dependencies = [
-            self._get_item_dependencies(CONNECTION, connection_id)
-            for connection_id in self.agent_config.connections
-        ]
-        skill_dependencies = [
-            self._get_item_dependencies(SKILL, skill_id)
-            for skill_id in self.agent_config.skills
-        ]
-        contract_dependencies = [
-            self._get_item_dependencies(CONTRACT, contract_id)
-            for contract_id in self.agent_config.contracts
-        ]
-
-        all_dependencies = [
-            self.agent_config.dependencies,
-            *protocol_dependencies,
-            *connection_dependencies,
-            *skill_dependencies,
-            *contract_dependencies,
-        ]
-
-        result = merge_dependencies_list(*all_dependencies)
-        return result
+        dependencies: Dependencies = {}
+
+        def _update_dependencies(updates: Dependencies) -> None:
+            """Update dependencies."""
+            for dep, spec in updates.items():
+                if dep in dependencies and dependencies[dep] != spec:
+                    logger.debug(
+                        f"`{dependencies[dep].get_pip_install_args()}` "
+                        f"will be overridden by {spec.get_pip_install_args()}"
+                    )
+                dependencies[dep] = spec
+
+        for item_type in (
+            PackageType.PROTOCOL,
+            PackageType.CONTRACT,
+            PackageType.CONNECTION,
+            PackageType.SKILL,
+            PackageType.AGENT,
+        ):
+            logger.debug(f"Loading {item_type.value} dependencies")
+            type_deps = self._get_dependencies_by_item_type(item_type)
+            _update_dependencies(type_deps)
+
+        if extra_dependencies is not None and len(extra_dependencies) > 0:
+            logger.debug("Loading extra dependencies")
+            type_deps = {spec.name: spec for spec in extra_dependencies}
+            _update_dependencies(type_deps)
+
+        return dependencies
 
     def dump_agent_config(self) -> None:
         """Dump the current agent configuration."""

aea/configurations/data_types.py

@@ -62,6 +62,9 @@
 T = TypeVar("T")
 PackageVersionLike = Union[str, semver.VersionInfo]
 
+PYPI_RE = r"(?P<name>[a-zA-Z0-9_-]+)(?P<version>(>|<|=|~).+)?"
+GIT_RE = r"git\+(?P<git>https://github.com/([a-z-_0-9A-Z]+\/[a-z-_0-9A-Z]+)\.git)@(?P<ref>.+)#egg=(?P<name>.+)"
+
 
 class JSONSerializable(ABC):
     """Interface for JSON-serializable objects."""
@@ -838,6 +841,28 @@ def _parse_version(version: Union[str, SpecifierSet]) -> SpecifierSet:
         """
         return version if isinstance(version, SpecifierSet) else SpecifierSet(version)
 
+    @classmethod
+    def from_string(cls, string: str) -> "Dependency":
+        """Parse from string."""
+        match = re.match(GIT_RE, string)
+        if match is not None:
+            data = match.groupdict()
+            return cls(name=data["name"], git=data["git"], ref=data["ref"])
+
+        match = re.match(PYPI_RE, string)
+        if match is None:
+            raise ValueError(f"Cannot parse the dependency string '{string}'")
+
+        data = match.groupdict()
+        return Dependency(
+            name=data["name"],
+            version=(
+                SpecifierSet(data["version"])
+                if data["version"] is not None
+                else SpecifierSet("")
+            ),
+        )
+
     @classmethod
     def from_json(cls, obj: Dict[str, Dict[str, str]]) -> "Dependency":
         """Parse a dependency object from a dictionary."""

docs/api/configurations/data_types.md

@@ -1067,6 +1067,17 @@ def ref() -> Optional[str]
 
 Get the ref.
 
+<a id="aea.configurations.data_types.Dependency.from_string"></a>
+
+#### from`_`string
+
+```python
+@classmethod
+def from_string(cls, string: str) -> "Dependency"
+```
+
+Parse from string.
+
 <a id="aea.configurations.data_types.Dependency.from_json"></a>
 
 #### from`_`json