Merge pull request #665 from valory-xyz/fix/remove-forked-dependencies

Remove forked dependencies
valory-xyz · Sep 26, 2023 · c4be6cd · c4be6cd
2 parents 391552e + e5fc97e
commit c4be6cd
Show file tree

Hide file tree

Showing 148 changed files with 1,473 additions and 7,026 deletions.
diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
@@ -123,8 +123,8 @@ jobs:
           pip install tomte[tox]==0.2.13
           pip install --user --upgrade setuptools
           # install Protobuf compiler
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_64.zip
-          unzip protoc-3.19.4-linux-x86_64.zip -d protoc
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
+          unzip protoc-24.3-linux-x86_64.zip -d protoc
           sudo mv protoc/bin/protoc /usr/local/bin/protoc
           # install IPFS
           sudo apt-get install -y wget
@@ -347,8 +347,8 @@ jobs:
           sudo apt-get autoclean
           pip install tomte[tox]==0.2.13
           # install Protobuf compiler
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_64.zip
-          unzip protoc-3.19.4-linux-x86_64.zip -d protoc
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
+          unzip protoc-24.3-linux-x86_64.zip -d protoc
           sudo mv protoc/bin/protoc /usr/local/bin/protoc
           
           # pull pre-built images
@@ -384,8 +384,8 @@ jobs:
           sudo apt-get autoclean
           pip install tomte[tox]==0.2.13
           # install Protobuf compiler
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_64.zip
-          unzip protoc-3.19.4-linux-x86_64.zip -d protoc
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
+          unzip protoc-24.3-linux-x86_64.zip -d protoc
           sudo mv protoc/bin/protoc /usr/local/bin/protoc
       - name: Framework integration tests
         run: tox -e py3.10 -- -m 'profiling'
@@ -450,8 +450,8 @@ jobs:
           sudo apt-get autoclean
           pip install tomte[tox]==0.2.13
           # install Protobuf compiler
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_64.zip
-          unzip protoc-3.19.4-linux-x86_64.zip -d protoc
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-linux-x86_64.zip
+          unzip protoc-24.3-linux-x86_64.zip -d protoc
           sudo mv protoc/bin/protoc /usr/local/bin/protoc
           make protolint_install
           # install ipfs
@@ -474,8 +474,8 @@ jobs:
           brew install gcc
           # brew install protobuf
           # brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/72457f0166d5619a83f508f2345b22d0617b5021/Formula/protobuf.rb
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-osx-x86_64.zip
-          unzip protoc-3.19.4-osx-x86_64.zip -d protoc
+          wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-osx-x86_64.zip
+          unzip protoc-24.3-osx-x86_64.zip -d protoc
           sudo mv protoc/bin/protoc /usr/local/bin/protoc
           brew tap yoheimuta/protolint
           brew install protolint
@@ -496,14 +496,14 @@ jobs:
         run: |
           python -m pip install -U pip
           echo "::add-path::C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64"
-          choco install protoc --version 3.19.4
+          choco install protoc --version 24.3
           choco install mingw -y
           choco install make -y
           # to check make was installed
           make --version
           pip install tomte[tox]==0.2.13
-          # wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-win64.zip
-          # unzip protoc-3.19.4-win64.zip -d protoc
+          # wget https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-win64.zip
+          # unzip protoc-24.3-win64.zip -d protoc
           # sudo mv protoc/bin/protoc /usr/local/bin/protoc
           python scripts/update_symlinks_cross_platform.py
           make protolint_install_win

diff --git a/Makefile b/Makefile
@@ -89,6 +89,7 @@ dist: clean
 	python setup.py bdist_wheel --plat-name=manylinux1_x86_64
 	python setup.py bdist_wheel --plat-name=manylinux2014_aarch64
 	python setup.py bdist_wheel --plat-name=macosx_10_9_x86_64
+	python setup.py bdist_wheel
 
 h := $(shell git rev-parse --abbrev-ref HEAD)
 
@@ -182,3 +183,17 @@ common-checks-1:
 common-checks-2:
 	tox -e check-api-docs
 	tox -e check-doc-links-hashes
+
+.PHONY: build-proto
+build-proto:
+	@# Usage: INCLUDE=PATH_TO_PROTOC_INCLUDE_DIRECTORY make build-proto
+	@echo "Building aea/mail/base.proto"
+	@protoc --proto_path=aea/mail/ --python_out=aea/mail/ aea/mail/base.proto -I $$INCLUDE
+	@echo "Building aea/ipfs/pb/merkledag.proto"
+	@protoc --proto_path=aea/helpers/ipfs/pb/ --python_out=aea/helpers/ipfs/pb/ aea/helpers/ipfs/pb/merkledag.proto -I $$INCLUDE
+	@echo "Building aea/ipfs/pb/unixfs.proto"
+	@protoc --proto_path=aea/helpers/ipfs/pb/ --python_out=aea/helpers/ipfs/pb/ aea/helpers/ipfs/pb/unixfs.proto -I $$INCLUDE
+	@echo "Building aea/helpers/search/models.proto"
+	@protoc --proto_path=aea/helpers/search/ --python_out=aea/helpers/search/ aea/helpers/search/models.proto -I $$INCLUDE
+	@echo "Building aea/helpers/multiaddr/crypto.proto"
+	@protoc --proto_path=aea/helpers/multiaddr/ --python_out=aea/helpers/multiaddr/ aea/helpers/multiaddr/crypto.proto -I $$INCLUDE
diff --git a/Pipfile b/Pipfile
@@ -31,15 +31,15 @@ openapi-core = "==0.13.2"
 openapi-spec-validator = "==0.2.8"
 packaging = ">=23.1,<24.0"
 pexpect = "==4.8.0"
-protobuf = "==3.19.5"
+protobuf = ">=4.21.6,<5.0.0"
 psutil = "==5.7.0"
 pycryptodome = ">=3.10.1"
 pytest-custom-exit-code = "==0.3.0"
 GitPython = "==3.1.27"
 requests = "==2.28.1"
 idna = "<=3.3"
-open-aea-cosmpy = "==0.6.5"
-open-aea-web3 = "==6.0.1"
+open-aea-cosmpy = "==0.6.6"
+web3 = ">=6.0.0,<7"
 semver = "<3.0.0,>=2.9.1"
 py-multibase = ">=1.0.0"
 py-multicodec = ">=0.2.0"

diff --git a/aea/__init__.py b/aea/__init__.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022 Valory AG
+#   Copyright 2022-2023 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,8 +20,23 @@
 
 """Contains the AEA package."""
 
+# pylint: disable=wrong-import-position
+
 import inspect
 import os
+import sys
+
+
+# Patch to fix https://github.com/protocolbuffers/protobuf/issues/3276
+
+_google_upb_message = sys.modules.pop("google._upb._message", None)
+
+from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
+
+
+if _google_upb_message is not None:
+    sys.modules["google._upb._message"] = _google_upb_message
+
 
 from packaging.version import Version
 

diff --git a/aea/cli/__init__.py b/aea/cli/__init__.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
+#   Copyright 2023 Valory AG
 #   Copyright 2018-2019 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,4 +20,5 @@
 
 """This module contains the implementation of command-line tool 'aea'."""
 
+
 from .core import cli
diff --git a/aea/cli/generate_all_protocols.py b/aea/cli/generate_all_protocols.py
@@ -63,7 +63,7 @@
 
 
 SPECIFICATION_REGEX = re.compile(r"(---\nname.*\.\.\.)", re.DOTALL)
-LIBPROTOC_VERSION = "libprotoc 3.19.4"
+LIBPROTOC_VERSION = "libprotoc 24.3"
 CUSTOM_TYPE_MODULE_NAME = "custom_types.py"
 PROTOCOL_GENERATOR_DOCSTRING_REGEX = "It was created with protocol buffer compiler version `libprotoc .*` and aea protocol generator version `.*`."
 

diff --git a/aea/configurations/constants.py b/aea/configurations/constants.py
@@ -30,7 +30,7 @@
 _SOLANA_IDENTIFIER = "solana"
 SIGNING_PROTOCOL = "open_aea/signing:latest"
 SIGNING_PROTOCOL_WITH_HASH = (
-    "open_aea/signing:1.0.0:bafybeifuxs7gdg2okbn7uofymenjlmnih2wxwkym44lsgwmklgwuckxm2m"
+    "open_aea/signing:1.0.0:bafybeie7xyems76v5b4wc2lmaidcujizpxfzjnnwdeokmhje53g7ym25ii"
 )
 DEFAULT_LEDGER = _ETHEREUM_IDENTIFIER
 PRIVATE_KEY_PATH_SCHEMA = "{}_private_key.txt"

diff --git a/aea/helpers/ipfs/__init__.py b/aea/helpers/ipfs/__init__.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022 Valory AG
+#   Copyright 2022-2023 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,10 +23,10 @@
 
 # fix for ipfs hashes, preload protobuf classes with protobuf python implementation
 with _protobuf_python_implementation():
-    from aea.helpers.ipfs.pb import (  # noqa: F401   # pylint: disable=import-outside-toplevel,unused-import
+    from aea.helpers.ipfs.pb import (  # type: ignore  # noqa: F401   # pylint: disable=import-outside-toplevel,unused-import
         merkledag_pb2,
         unixfs_pb2,
     )
-    from aea.helpers.ipfs.pb.merkledag_pb2 import (  # noqa: F401   # pylint: disable=import-outside-toplevel,unused-import
+    from aea.helpers.ipfs.pb.merkledag_pb2 import (  # type: ignore  # noqa: F401   # pylint: disable=import-outside-toplevel,unused-import
         PBNode,
     )
diff --git a/aea/helpers/ipfs/base.py b/aea/helpers/ipfs/base.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022 Valory AG
+#   Copyright 2022-2023 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -40,7 +40,7 @@
 
 with _protobuf_python_implementation():  # pylint: disable=import-outside-toplevel
     from aea.helpers.ipfs.pb import merkledag_pb2, unixfs_pb2
-    from aea.helpers.ipfs.pb.merkledag_pb2 import PBNode
+    from aea.helpers.ipfs.pb.merkledag_pb2 import PBNode  # type: ignore
 
 
 def _dos2unix(file_content: bytes) -> bytes:
@@ -179,7 +179,7 @@ def hash_directory(
     @staticmethod
     def create_link(link_hash: bytes, tsize: int, name: str) -> Any:
         """Create PBLink object."""
-        link = merkledag_pb2.PBLink()
+        link = merkledag_pb2.PBLink()  # type: ignore # pylint: disable=no-member
         link.Hash = link_hash
         link.Tsize = tsize
         link.Name = name
@@ -192,9 +192,9 @@ def wrap_in_a_node(cls, link: Any) -> str:
         wrapper_node = PBNode()
         wrapper_node.Links.append(link)  # type: ignore # pylint: disable=no-member
 
-        wrapper_node_data = unixfs_pb2.Data()
+        wrapper_node_data = unixfs_pb2.Data()  # type: ignore # pylint: disable=no-member
         # type: ignore  # pylint: disable=no-member
-        wrapper_node_data.Type = unixfs_pb2.Data.Directory
+        wrapper_node_data.Type = unixfs_pb2.Data.Directory  # type: ignore
         wrapper_node.Data = wrapper_node_data.SerializeToString(
             deterministic=True
         )  # type: ignore # pylint: disable=no-member
@@ -241,9 +241,9 @@ def _hash_directory_recursively(cls, root: Path) -> Dict:
                     cls.create_link(child_hash, file_length, child_path.name)
                 )
 
-        root_node_data = unixfs_pb2.Data()
+        root_node_data = unixfs_pb2.Data()  # type: ignore # pylint: disable=no-member
         # type: ignore  # pylint: disable=no-member
-        root_node_data.Type = unixfs_pb2.Data.Directory
+        root_node_data.Type = unixfs_pb2.Data.Directory  # type: ignore
         root_node.Data = root_node_data.SerializeToString(
             deterministic=True
         )  # type: ignore # pylint: disable=no-member
@@ -263,7 +263,7 @@ def _hash_directory_recursively(cls, root: Path) -> Dict:
     def _make_unixfs_pb2(cls, data: bytes) -> bytes:
         if len(data) > cls.DEFAULT_CHUNK_SIZE:  # pragma: nocover
             raise ValueError("Data is too big! use chunks!")
-        data_pb = unixfs_pb2.Data()  # type: ignore
+        data_pb = unixfs_pb2.Data()  # type: ignore  # pylint: disable=no-member
         data_pb.Type = unixfs_pb2.Data.File  # type: ignore # pylint: disable=no-member
         data_pb.Data = data
         data_pb.filesize = len(data)
@@ -299,8 +299,8 @@ def _pb_serialize_bytes(cls, data: bytes) -> Tuple[bytes, int]:
         if len(data) > cls.DEFAULT_CHUNK_SIZE:
             content_size = 0
             outer_node = PBNode()  # type: ignore
-            data_pb = unixfs_pb2.Data()  # type: ignore
-            data_pb.Type = unixfs_pb2.Data.File  # type: ignore # pylint: disable=no-member
+            data_pb = unixfs_pb2.Data()  # type: ignore   # pylint: disable=no-member
+            data_pb.Type = unixfs_pb2.Data.File  # type: ignore  # pylint: disable=no-member
             data_pb.filesize = len(data)
             for chunk in chunks(data, cls.DEFAULT_CHUNK_SIZE):
                 block = cls._pb_serialize_data(chunk)