diff --git a/.gitignore b/.gitignore index e9040821c1..87caa4be97 100644 --- a/.gitignore +++ b/.gitignore @@ -154,4 +154,5 @@ coverage.txt temp/ agent_name/ +agent/ leak_report diff --git a/HISTORY.md b/HISTORY.md index 0721adaa03..3d5fdefa8c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,12 @@ # Release History - open AEA +## 1.40.0 (2023-09-26) + +AEA: +- Adds support for specifying extra dependencies and overriding dependencies via `-e` flag on `aea install` +- Updates the selection of dependencies in `aea install` command to override the dependencies in the `extra dependencies provided by flag > agent > skill > connection > contract > protocol` order instead of merging them. + ## 1.40.0 (2023-09-26) AEA: diff --git a/aea/cli/install.py b/aea/cli/install.py index 7bf2bb1f89..e4b728178b 100644 --- a/aea/cli/install.py +++ b/aea/cli/install.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2022 Valory AG +# Copyright 2022-2023 Valory AG # Copyright 2018-2021 Fetch.AI Limited # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,15 +19,15 @@ # ------------------------------------------------------------------------------ """Implementation of the 'aea install' subcommand.""" -from typing import Optional, cast +from typing import Optional, Tuple, cast import click +from aea.cli.utils.click_utils import PyPiDependency from aea.cli.utils.context import Context from aea.cli.utils.decorators import check_aea_project from aea.cli.utils.loggers import logger -from aea.configurations.data_types import Dependencies -from aea.configurations.pypi import is_satisfiable, is_simple_dep, to_set_specifier +from aea.configurations.data_types import Dependency from aea.exceptions import AEAException from aea.helpers.install_dependency import call_pip, install_dependencies @@ -41,66 +41,56 @@ default=None, help="Install from the given requirements file.", ) +@click.option( + "-e", + "--extra-dependency", + "extra_dependencies", + type=PyPiDependency(), + help="Provide extra dependency.", + multiple=True, +) @click.pass_context @check_aea_project -def install(click_context: click.Context, requirement: Optional[str]) -> None: +def install( + click_context: click.Context, + requirement: Optional[str], + extra_dependencies: Tuple[Dependency], +) -> None: """Install the dependencies of the agent.""" ctx = cast(Context, click_context.obj) - do_install(ctx, requirement) + do_install(ctx, requirement, extra_dependencies) -def do_install(ctx: Context, requirement: Optional[str] = None) -> None: +def do_install( + ctx: Context, + requirement: Optional[str] = None, + extra_dependencies: Optional[Tuple[Dependency]] = None, +) -> None: """ Install necessary dependencies. :param ctx: context object. :param requirement: optional str requirement. + :param extra_dependencies: List of the extra dependencies to use :raises ClickException: if AEAException occurs. """ try: if requirement: + if extra_dependencies is not None and len(extra_dependencies) > 0: + logger.debug( + "Extra dependencies will be ignored while installing from requirements file" + ) logger.debug("Installing the dependencies in '{}'...".format(requirement)) _install_from_requirement(requirement) else: logger.debug("Installing all the dependencies...") - dependencies = ctx.get_dependencies() - - logger.debug("Preliminary check on satisfiability of version specifiers...") - unsat_dependencies = _find_unsatisfiable_dependencies(dependencies) - if len(unsat_dependencies) != 0: - raise AEAException( - "cannot install the following dependencies " - + "as the joint version specifier is unsatisfiable:\n - " - + "\n -".join( - [ - f"{name}: {to_set_specifier(dep)}" - for name, dep in unsat_dependencies.items() - ] - ) - ) + dependencies = ctx.get_dependencies(extra_dependencies=extra_dependencies) install_dependencies(list(dependencies.values()), logger=logger) except AEAException as e: raise click.ClickException(str(e)) -def _find_unsatisfiable_dependencies(dependencies: Dependencies) -> Dependencies: - """ - Find unsatisfiable dependencies. - - It only checks among 'simple' dependencies (i.e. if it has no field specified, - or only the 'version' field set.) - - :param dependencies: the dependencies to check. - :return: the unsatisfiable dependencies. - """ - return { - name: dep - for name, dep in dependencies.items() - if is_simple_dep(dep) and not is_satisfiable(to_set_specifier(dep)) - } - - def _install_from_requirement(file: str, install_timeout: float = 300) -> None: """ Install from requirements. diff --git a/aea/cli/utils/click_utils.py b/aea/cli/utils/click_utils.py index 18d81d64fd..9c44132045 100644 --- a/aea/cli/utils/click_utils.py +++ b/aea/cli/utils/click_utils.py @@ -45,7 +45,7 @@ PROTOCOL, SKILL, ) -from aea.configurations.data_types import PackageType, PublicId +from aea.configurations.data_types import Dependency, PackageType, PublicId from aea.helpers.io import open_file from aea.package_manager.base import PACKAGE_SOURCE_RE @@ -244,6 +244,21 @@ def convert(self, value: str, param: Any, ctx: click.Context) -> str: return value +class PyPiDependency(click.ParamType): + """Click parameter for PyPy dependency string""" + + def get_metavar(self, param: Any) -> str: + """Return the metavar default for this param if it provides one.""" + return "DEPENDENCY" # pragma: no cover + + def convert(self, value: str, param: Any, ctx: click.Context) -> Dependency: + """Convert the value.""" + try: + return Dependency.from_string(value) + except ValueError as e: + raise click.ClickException(str(e)) from e + + def registry_flag( mark_default: bool = True, default_registry: Optional[str] = None, diff --git a/aea/cli/utils/context.py b/aea/cli/utils/context.py index b68a96be9a..34613ca657 100644 --- a/aea/cli/utils/context.py +++ b/aea/cli/utils/context.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2022 Valory AG +# Copyright 2022-2023 Valory AG # Copyright 2018-2021 Fetch.AI Limited # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,28 +20,31 @@ """A module with context tools of the aea cli.""" import os from pathlib import Path -from typing import Any, Dict, List, Optional, cast +from typing import Any, Dict, List, Optional, Tuple, cast from aea.cli.registry.settings import REGISTRY_LOCAL, REGISTRY_TYPES from aea.cli.utils.loggers import logger from aea.configurations.base import ( AgentConfig, Dependencies, + Dependency, PackageType, PublicId, _get_default_configuration_file_name_from_type, ) from aea.configurations.constants import ( - CONNECTION, - CONTRACT, DEFAULT_AEA_CONFIG_FILE, DEFAULT_REGISTRY_NAME, - PROTOCOL, - SKILL, VENDOR, ) from aea.configurations.loader import ConfigLoader -from aea.configurations.pypi import merge_dependencies_list +from aea.configurations.pypi import ( + is_satisfiable, + is_simple_dep, + merge_dependencies_list, + to_set_specifier, +) +from aea.exceptions import AEAException from aea.helpers.io import open_file @@ -184,39 +187,96 @@ def _get_item_dependencies(item_type: str, public_id: PublicId) -> Dependencies: deps = cast(Dependencies, config.dependencies) return deps - def get_dependencies(self) -> Dependencies: + @staticmethod + def _find_unsatisfiable_dependencies(dependencies: Dependencies) -> Dependencies: + """ + Find unsatisfiable dependencies. + + It only checks among 'simple' dependencies (i.e. if it has no field specified, + or only the 'version' field set.) + + :param dependencies: the dependencies to check. + :return: the unsatisfiable dependencies. + """ + return { + name: dep + for name, dep in dependencies.items() + if is_simple_dep(dep) and not is_satisfiable(to_set_specifier(dep)) + } + + def _get_dependencies_by_item_type(self, item_type: PackageType) -> Dependencies: + """Get the dependencies from item type and public id.""" + if item_type == PackageType.AGENT: + return self.agent_config.dependencies + dependency_to_package: Dict[str, List[Tuple[PublicId, Dependency]]] = {} + dependencies = [] + for item_id in getattr(self.agent_config, item_type.to_plural()): + package_dependencies = self._get_item_dependencies(item_type.value, item_id) + dependencies += [package_dependencies] + for dep, spec in package_dependencies.items(): + if dep not in dependency_to_package: + dependency_to_package[dep] = [] + dependency_to_package[dep].append((item_id, spec)) + + merged_dependencies = merge_dependencies_list(*dependencies) + unsat_dependencies = self._find_unsatisfiable_dependencies(merged_dependencies) + if len(unsat_dependencies) > 0: + error = f"Error while merging dependencies for {item_type.to_plural()}" + error += "; Joint version specifier is unsatisfiable for following dependencies:\n" + error += "======================================\n" + for name, spec in unsat_dependencies.items(): + error += f"Dependency: {name}\n" + error += f"Specifier: {to_set_specifier(spec)}\n" + error += "Packages containing dependency: \n" + for package, dep_spec in dependency_to_package[name]: + error += f" - {package.without_hash()}: {dep_spec.get_pip_install_args()[0]}\n" + error += "======================================\n" + + raise AEAException(error[:-1]) + return merged_dependencies + + def get_dependencies( + self, + extra_dependencies: Optional[Tuple[Dependency]] = None, + ) -> Dependencies: """ Aggregate the dependencies from every component. + :param extra_dependencies: List of the extra dependencies to use, if the + extra dependencies and agent dependencies have conflicts + the packages from extra dependencies list will be prefered + over the agent dependencies :return: a list of dependency version specification. e.g. ["gym >= 1.0.0"] """ - protocol_dependencies = [ - self._get_item_dependencies(PROTOCOL, protocol_id) - for protocol_id in self.agent_config.protocols - ] - connection_dependencies = [ - self._get_item_dependencies(CONNECTION, connection_id) - for connection_id in self.agent_config.connections - ] - skill_dependencies = [ - self._get_item_dependencies(SKILL, skill_id) - for skill_id in self.agent_config.skills - ] - contract_dependencies = [ - self._get_item_dependencies(CONTRACT, contract_id) - for contract_id in self.agent_config.contracts - ] - - all_dependencies = [ - self.agent_config.dependencies, - *protocol_dependencies, - *connection_dependencies, - *skill_dependencies, - *contract_dependencies, - ] - - result = merge_dependencies_list(*all_dependencies) - return result + dependencies: Dependencies = {} + + def _update_dependencies(updates: Dependencies) -> None: + """Update dependencies.""" + for dep, spec in updates.items(): + if dep in dependencies and dependencies[dep] != spec: + logger.debug( + f"`{dependencies[dep].get_pip_install_args()}` " + f"will be overridden by {spec.get_pip_install_args()}" + ) + dependencies[dep] = spec + + for item_type in ( + PackageType.PROTOCOL, + PackageType.CONTRACT, + PackageType.CONNECTION, + PackageType.SKILL, + PackageType.AGENT, + ): + logger.debug(f"Loading {item_type.value} dependencies") + type_deps = self._get_dependencies_by_item_type(item_type) + _update_dependencies(type_deps) + + if extra_dependencies is not None and len(extra_dependencies) > 0: + logger.debug("Loading extra dependencies") + type_deps = {spec.name: spec for spec in extra_dependencies} + _update_dependencies(type_deps) + + return dependencies def dump_agent_config(self) -> None: """Dump the current agent configuration.""" diff --git a/aea/configurations/data_types.py b/aea/configurations/data_types.py index d53609bc38..78d4da52e1 100644 --- a/aea/configurations/data_types.py +++ b/aea/configurations/data_types.py @@ -62,6 +62,9 @@ T = TypeVar("T") PackageVersionLike = Union[str, semver.VersionInfo] +PYPI_RE = r"(?P[a-zA-Z0-9_-]+)(?P(>|<|=|~).+)?" +GIT_RE = r"git\+(?Phttps://github.com/([a-z-_0-9A-Z]+\/[a-z-_0-9A-Z]+)\.git)@(?P.+)#egg=(?P.+)" + class JSONSerializable(ABC): """Interface for JSON-serializable objects.""" @@ -838,6 +841,28 @@ def _parse_version(version: Union[str, SpecifierSet]) -> SpecifierSet: """ return version if isinstance(version, SpecifierSet) else SpecifierSet(version) + @classmethod + def from_string(cls, string: str) -> "Dependency": + """Parse from string.""" + match = re.match(GIT_RE, string) + if match is not None: + data = match.groupdict() + return cls(name=data["name"], git=data["git"], ref=data["ref"]) + + match = re.match(PYPI_RE, string) + if match is None: + raise ValueError(f"Cannot parse the dependency string '{string}'") + + data = match.groupdict() + return Dependency( + name=data["name"], + version=( + SpecifierSet(data["version"]) + if data["version"] is not None + else SpecifierSet("") + ), + ) + @classmethod def from_json(cls, obj: Dict[str, Dict[str, str]]) -> "Dependency": """Parse a dependency object from a dictionary.""" diff --git a/docs/api/configurations/data_types.md b/docs/api/configurations/data_types.md index a0e112262c..5dc85a362e 100644 --- a/docs/api/configurations/data_types.md +++ b/docs/api/configurations/data_types.md @@ -1067,6 +1067,17 @@ def ref() -> Optional[str] Get the ref. + + +#### from`_`string + +```python +@classmethod +def from_string(cls, string: str) -> "Dependency" +``` + +Parse from string. + #### from`_`json diff --git a/docs/upgrading.md b/docs/upgrading.md index 49622181a5..1aa71850be 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -9,6 +9,16 @@ Below we describe the additional manual steps required to upgrade between differ ### Upgrade guide +## `v1.40.0` to `v1.40.1` + +- The way the dependencies will be selected for installation when running `aea install` has changed. Before this version, the versions were being merging all of the versions for a python package and using the most compatible version specifier possible. With this release, this behaviour will be replaced by overriding the dependencies in the following order `extra dependencies provided by flag > agent > skill > connection > contract > protocol` what this means is, let's say you have 3 packages with a same python package as a dependency + +* protocol package with `protobuf>1.0.0` +* connection package with `protobuf==1.0.0` +* skill package with `protobuf>=1.0.0,<2.0.0` + +`protobuf>=1.0.0,<2.0.0` will be used for installation since skill has higher priority over protocol and connection packages. + ## `v1.39.0.post1` to `v1.40.0` - `open-aea-web3` has been replaced with `web3py` diff --git a/tests/test_cli/test_install.py b/tests/test_cli/test_install.py index 25a4519559..036d2d1888 100644 --- a/tests/test_cli/test_install.py +++ b/tests/test_cli/test_install.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2022 Valory AG +# Copyright 2022-2023 Valory AG # Copyright 2018-2021 Fetch.AI Limited # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,8 +20,10 @@ """This test module contains the tests for the `aea install` sub-command.""" +import logging from pathlib import Path -from typing import Dict +from typing import Any, Dict +from unittest import mock import pytest import yaml @@ -39,15 +41,30 @@ class TestInstall(AEATestCase): path_to_aea: Path = Path(CUR_PATH, "data", "dummy_aea") - @classmethod - def setup_class(cls): - """Set the test up.""" - super().setup_class() - cls.result = cls.run_cli_command("install", cwd=cls._get_cwd()) - def test_exit_code_equal_to_zero(self): """Assert that the exit code is equal to zero (i.e. success).""" - assert self.result.exit_code == 0 + result = self.run_cli_command("install", cwd=self._get_cwd()) + assert result.exit_code == 0 + + def test_extra_dependencies(self, caplog: Any): + """Assert that the exit code is equal to zero (i.e. success).""" + with caplog.at_level(logging.DEBUG), mock.patch( + "aea.cli.install.install_dependencies" + ): + result = self.run_cli_command( + "-v", + "DEBUG", + "install", + "-e", + "open-aea-ledger-cosmos==1.0.0", + cwd=self._get_cwd(), + ) + + assert result.exit_code == 0 + assert ( + "`['open-aea-ledger-cosmos<2.0.0,>=1.0.0']` will be overridden by ['open-aea-ledger-cosmos==1.0.0']" + in caplog.text + ) class TestInstallFromRequirementFile(AEATestCase): @@ -179,6 +196,6 @@ def test_error(self): """Assert an error occurs.""" with pytest.raises( ClickException, - match="cannot install the following dependencies as the joint version specifier is unsatisfiable:\n - this_is_a_test_dependency: ==0.1.0,==0.2.0", + match="Error while merging dependencies for connections; Joint version specifier is unsatisfiable for following dependencies", ): self.run_cli_command("install", cwd=self._get_cwd())