From dd173665a129c729b86a2b6d97dc9975fb32c98d Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Thu, 5 Sep 2024 13:25:35 +0530 Subject: [PATCH 01/13] added minio config Signed-off-by: Pratiksha Sankhe --- .../api/elixircloud/csh/models.py | 40 ++++++ cloud_storage_handler/consts.py | 27 ++++ cloud_storage_handler/custom_config.py | 12 ++ cloud_storage_handler/exceptions.py | 5 + cloud_storage_handler/main.py | 1 + cloud_storage_handler/minio_client.py | 74 +++++++++++ cloud_storage_handler/utils.py | 37 ++++++ deployment/config.yaml | 10 ++ ...ud_storage_handler.api.elixircloud.csh.rst | 8 ++ docs/source/pages/cloud_storage_handler.rst | 32 +++++ poetry.lock | 118 +++++++++++++++++- pyproject.toml | 2 + 12 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 cloud_storage_handler/api/elixircloud/csh/models.py create mode 100644 cloud_storage_handler/consts.py create mode 100644 cloud_storage_handler/custom_config.py create mode 100644 cloud_storage_handler/minio_client.py create mode 100644 cloud_storage_handler/utils.py diff --git a/cloud_storage_handler/api/elixircloud/csh/models.py b/cloud_storage_handler/api/elixircloud/csh/models.py new file mode 100644 index 0000000..2768b10 --- /dev/null +++ b/cloud_storage_handler/api/elixircloud/csh/models.py @@ -0,0 +1,40 @@ +"""Model for MinIO Configuration.""" + +from typing import Annotated + +from pydantic import BaseModel, Field, constr + + +class MinioConfig(BaseModel): + """Configuration for MinIO.""" + + hostname: str = Field( + ..., + description="The hostname where the MinIO server is running.", + example="localhost", + ) + port: int = Field( + ..., description="The port on which the MinIO server is running.", example=9000 + ) + access_key: str = Field( + ..., + description="The access key used for authentication with MinIO.", + example="minioadmin", + ) + secret_key: str = Field( + ..., + description="The secret key used for authentication with MinIO.", + example="minioadmin", + ) + is_secure: bool = Field( + ..., + description=( + "Specifies whether the connection to MinIO should be secure (HTTPS)." + ), + example=False, + ) + bucket_name: Annotated[str, constr(min_length=1)] = Field( + ..., + description="The name of the bucket where files are stored.", + example="files", + ) diff --git a/cloud_storage_handler/consts.py b/cloud_storage_handler/consts.py new file mode 100644 index 0000000..5f5da72 --- /dev/null +++ b/cloud_storage_handler/consts.py @@ -0,0 +1,27 @@ +"""This module handles the loading of environment variables for MinIO configuration.""" + +from cloud_storage_handler.minio_client import MinioClient + + +def GetMinioConfiguration(): # type: ignore + """Get minio configuration.""" + minio_config = MinioClient() + return minio_config.response() + + +minio_config_data = GetMinioConfiguration() + +MINIO_HOSTNAME = minio_config_data["hostname"] +MINIO_PORT = minio_config_data["port"] +MINIO_ENDPOINT = f"{MINIO_HOSTNAME}:{MINIO_PORT}" +MINIO_SECURE = minio_config_data["is_secure"] + +minio_client_instance = MinioClient() +minio_client = minio_client_instance.initialise_minio( + endpoint=MINIO_ENDPOINT, + access_key=minio_config_data["access_key"], + secret_key=minio_config_data["secret_key"], + secure=MINIO_SECURE, +) + +minio_client_instance.create_bucket(minio_config_data["bucket_name"]) diff --git a/cloud_storage_handler/custom_config.py b/cloud_storage_handler/custom_config.py new file mode 100644 index 0000000..3f181cc --- /dev/null +++ b/cloud_storage_handler/custom_config.py @@ -0,0 +1,12 @@ +"""Custom configuration model for the FOCA app.""" + +from pydantic import BaseModel + +from cloud_storage_handler.api.elixircloud.csh.models import MinioConfig + + +class CustomConfig(BaseModel): + """Custom configuration model for the FOCA app.""" + + # Define custom configuration fields here + minio: MinioConfig diff --git a/cloud_storage_handler/exceptions.py b/cloud_storage_handler/exceptions.py index f5fb8c5..1b65658 100644 --- a/cloud_storage_handler/exceptions.py +++ b/cloud_storage_handler/exceptions.py @@ -4,6 +4,11 @@ from werkzeug.exceptions import BadRequest, InternalServerError, NotFound + +class ConfigNotFoundError(FileNotFoundError): + """Configuration file not found error.""" + + exceptions = { Exception: { "message": "An unexpected error occurred. Please try again.", diff --git a/cloud_storage_handler/main.py b/cloud_storage_handler/main.py index c7d499c..4f8fcf0 100644 --- a/cloud_storage_handler/main.py +++ b/cloud_storage_handler/main.py @@ -39,6 +39,7 @@ def init_app() -> FlaskApp: foca = Foca( config_file=config_path, + custom_config_model="cloud_storage_handler.custom_config.CustomConfig", ) return foca.create_app() diff --git a/cloud_storage_handler/minio_client.py b/cloud_storage_handler/minio_client.py new file mode 100644 index 0000000..a1e5948 --- /dev/null +++ b/cloud_storage_handler/minio_client.py @@ -0,0 +1,74 @@ +"""This module provides functionality for managing MinIO client.""" + +import logging + +from foca import Foca +from minio import Minio + +from cloud_storage_handler.exceptions import ConfigNotFoundError +from cloud_storage_handler.utils import get_config_path + +logger = logging.getLogger(__name__) + + +class MinioClient: + """A class to manage MinIO client configuration and bucket creation.""" + + def __init__(self) -> None: + """Initializes the MinIO client class.""" + self.minio_config_data = None + self.config = Foca(config_file=get_config_path()).conf + self.client = None # MinIO client will be initialized later + + def initialise_minio(self, endpoint, access_key, secret_key, secure): + """Initialize the MinIO client with provided configurations.""" + self.client = Minio( + endpoint=endpoint, + access_key=access_key, + secret_key=secret_key, + secure=secure, # Correctly use the secure flag + ) + return self.client + + def create_bucket(self, bucket_name): + """Create a new bucket if it does not already exist.""" + if self.client is None: + raise RuntimeError("MinIO client is not initialized.") + + if not self.client.bucket_exists(bucket_name): + self.client.make_bucket(bucket_name) + + def get_default_minio_config(self): + """Get the default minio configuration.""" + logger.warning( + "Minio configuration not found in config. Using default configuration." + ) + return { + "hostname": "localhost", + "port": 9000, + "access_key": "minioadmin", + "secret_key": "minioadmin", + "is_secure": False, + "bucket_name": "files", + } + + def get_minio_from_config(self): + """Returns minio configuration from config.""" + if not self.config.custom: + raise ConfigNotFoundError("Custom configuration not found.") + + minio_config_data = self.config.custom.get("minio") + if not minio_config_data: + raise ConfigNotFoundError("Service info not found in custom configuration.") + + return minio_config_data + + def response(self): + """Returns minio configuration response.""" + if self.minio_config_data is None: + try: + self.minio_config_data = self.get_minio_from_config() + except ConfigNotFoundError: + self.minio_config_data = self.get_default_minio_config() + + return self.minio_config_data diff --git a/cloud_storage_handler/utils.py b/cloud_storage_handler/utils.py new file mode 100644 index 0000000..7410cd5 --- /dev/null +++ b/cloud_storage_handler/utils.py @@ -0,0 +1,37 @@ +"""Utility functions for the Cloud Storage Handler package.""" + +import os +from pathlib import Path + +from foca import Foca + +from cloud_storage_handler.custom_config import CustomConfig +from cloud_storage_handler.exceptions import ConfigNotFoundError + + +def get_config_path() -> Path: + """Get the configuration path. + + Returns: + The path of the config file. + """ + # Determine the configuration path + if config_path_env := os.getenv("CSH_FOCA_CONFIG_PATH"): + return Path(config_path_env).resolve() + else: + return (Path(__file__).parents[1] / "deployment" / "config.yaml").resolve() + + +def get_custom_config() -> CustomConfig: + """Get the custom configuration. + + Returns: + The custom configuration. + """ + conf = Foca(config_file=get_config_path()).conf + try: + return CustomConfig(**conf.custom) + except AttributeError: + raise ConfigNotFoundError( + "Custom configuration not found in config file." + ) from None diff --git a/deployment/config.yaml b/deployment/config.yaml index 2d1329f..f322f77 100644 --- a/deployment/config.yaml +++ b/deployment/config.yaml @@ -35,6 +35,16 @@ api: swagger_ui: true serve_spec: true +custom: + # MinIO configuration based on MinIO model + minio: + hostname: localhost + port: 9001 + access_key: minioadmin + secret_key: minioadmin + is_secure: false + bucket_name: 'files' + exceptions: required_members: [['message'], ['code']] status_member: ['code'] diff --git a/docs/source/pages/cloud_storage_handler.api.elixircloud.csh.rst b/docs/source/pages/cloud_storage_handler.api.elixircloud.csh.rst index 2222742..8cc012a 100644 --- a/docs/source/pages/cloud_storage_handler.api.elixircloud.csh.rst +++ b/docs/source/pages/cloud_storage_handler.api.elixircloud.csh.rst @@ -12,6 +12,14 @@ cloud\_storage\_handler.api.elixircloud.csh.controllers module :undoc-members: :show-inheritance: +cloud\_storage\_handler.api.elixircloud.csh.models module +--------------------------------------------------------- + +.. automodule:: cloud_storage_handler.api.elixircloud.csh.models + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/source/pages/cloud_storage_handler.rst b/docs/source/pages/cloud_storage_handler.rst index dd44edb..156c83c 100644 --- a/docs/source/pages/cloud_storage_handler.rst +++ b/docs/source/pages/cloud_storage_handler.rst @@ -12,6 +12,22 @@ Subpackages Submodules ---------- +cloud\_storage\_handler.consts module +------------------------------------- + +.. automodule:: cloud_storage_handler.consts + :members: + :undoc-members: + :show-inheritance: + +cloud\_storage\_handler.custom\_config module +--------------------------------------------- + +.. automodule:: cloud_storage_handler.custom_config + :members: + :undoc-members: + :show-inheritance: + cloud\_storage\_handler.exceptions module ----------------------------------------- @@ -28,6 +44,22 @@ cloud\_storage\_handler.main module :undoc-members: :show-inheritance: +cloud\_storage\_handler.minio\_client module +-------------------------------------------- + +.. automodule:: cloud_storage_handler.minio_client + :members: + :undoc-members: + :show-inheritance: + +cloud\_storage\_handler.utils module +------------------------------------ + +.. automodule:: cloud_storage_handler.utils + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/poetry.lock b/poetry.lock index 0d81b95..ef24a6a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -60,6 +60,63 @@ files = [ [package.dependencies] vine = ">=5.0.0,<6.0.0" +[[package]] +name = "argon2-cffi" +version = "23.1.0" +description = "Argon2 for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "argon2_cffi-23.1.0-py3-none-any.whl", hash = "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"}, + {file = "argon2_cffi-23.1.0.tar.gz", hash = "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08"}, +] + +[package.dependencies] +argon2-cffi-bindings = "*" + +[package.extras] +dev = ["argon2-cffi[tests,typing]", "tox (>4)"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-notfound-page"] +tests = ["hypothesis", "pytest"] +typing = ["mypy"] + +[[package]] +name = "argon2-cffi-bindings" +version = "21.2.0" +description = "Low-level CFFI bindings for Argon2" +optional = false +python-versions = ">=3.6" +files = [ + {file = "argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_i686.whl", hash = "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win32.whl", hash = "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082"}, + {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f"}, + {file = "argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3e385d1c39c520c08b53d63300c3ecc28622f076f4c2b0e6d7e796e9f6502194"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3e3cc67fdb7d82c4718f19b4e7a87123caf8a93fde7e23cf66ac0337d3cb3f"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a22ad9800121b71099d0fb0a65323810a15f2e292f2ba450810a7316e128ee5"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351"}, + {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:93f9bf70084f97245ba10ee36575f0c3f1e7d7724d67d8e5b08e61787c320ed7"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3b9ef65804859d335dc6b31582cad2c5166f0c3e7975f324d9ffaa34ee7e6583"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4966ef5848d820776f5f562a7d45fdd70c2f330c961d0d745b784034bd9f48d"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ef543a89dee4db46a1a6e206cd015360e5a75822f76df533845c3cbaf72670"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed2937d286e2ad0cc79a7087d3c272832865f779430e0cc2b4f3718d3159b0cb"}, + {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5e00316dabdaea0b2dd82d141cc66889ced0cdcbfa599e8b471cf22c620c329a"}, +] + +[package.dependencies] +cffi = ">=1.0.1" + +[package.extras] +dev = ["cogapp", "pre-commit", "pytest", "wheel"] +tests = ["pytest"] + [[package]] name = "arrow" version = "1.3.0" @@ -1324,6 +1381,24 @@ files = [ {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] +[[package]] +name = "minio" +version = "7.2.8" +description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage" +optional = false +python-versions = ">3.8" +files = [ + {file = "minio-7.2.8-py3-none-any.whl", hash = "sha256:aa3b485788b63b12406a5798465d12a57e4be2ac2a58a8380959b6b748e64ddd"}, + {file = "minio-7.2.8.tar.gz", hash = "sha256:f8af2dafc22ebe1aef3ac181b8e217037011c430aa6da276ed627e55aaf7c815"}, +] + +[package.dependencies] +argon2-cffi = "*" +certifi = "*" +pycryptodome = "*" +typing-extensions = "*" +urllib3 = "*" + [[package]] name = "mistune" version = "0.8.4" @@ -1554,6 +1629,47 @@ files = [ {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] +[[package]] +name = "pycryptodome" +version = "3.20.0" +description = "Cryptographic library for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pycryptodome-3.20.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-win32.whl", hash = "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690"}, + {file = "pycryptodome-3.20.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc"}, + {file = "pycryptodome-3.20.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818"}, + {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"}, + {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"}, + {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"}, + {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"}, + {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"}, + {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"}, + {file = "pycryptodome-3.20.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a"}, + {file = "pycryptodome-3.20.0-pp27-pypy_73-win32.whl", hash = "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea"}, + {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5"}, + {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e"}, + {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"}, +] + [[package]] name = "pydantic" version = "1.10.17" @@ -2650,4 +2766,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<4.0" -content-hash = "43ba118188416b06b0408c02bfd4def3b77f632639bf22d168dfb645cc323c36" +content-hash = "f545068e708ae29fedfbc493beb007cfc10eae9cbd54d5980d74b8f7bf133c3e" diff --git a/pyproject.toml b/pyproject.toml index d5c83ed..a2e038d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ ignore_missing_imports = true module = [ "connexion.*", "foca", + "minio", ] [tool.poetry] @@ -32,6 +33,7 @@ version = "0.1.0" [tool.poetry.dependencies] flask = "2.2.3" foca = "^0.13.0" +minio = "^7.2.8" python = ">=3.11,<4.0" sphinx = "^8.0.2" From 8a68e43398f236e6a99209b56b6afdd64177b721 Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Thu, 5 Sep 2024 13:28:20 +0530 Subject: [PATCH 02/13] change Signed-off-by: Pratiksha Sankhe --- cloud_storage_handler/minio_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud_storage_handler/minio_client.py b/cloud_storage_handler/minio_client.py index a1e5948..aab32e5 100644 --- a/cloud_storage_handler/minio_client.py +++ b/cloud_storage_handler/minio_client.py @@ -26,7 +26,7 @@ def initialise_minio(self, endpoint, access_key, secret_key, secure): endpoint=endpoint, access_key=access_key, secret_key=secret_key, - secure=secure, # Correctly use the secure flag + secure=secure, ) return self.client From 4bb92d42f16b88d9bb114c713be68881823ce297 Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Thu, 5 Sep 2024 23:31:15 +0530 Subject: [PATCH 03/13] code refactoring Signed-off-by: Pratiksha Sankhe --- .../api/elixircloud/csh/models.py | 6 +- cloud_storage_handler/consts.py | 53 +++++++---- cloud_storage_handler/minio_client.py | 92 ++++++++++++------- 3 files changed, 98 insertions(+), 53 deletions(-) diff --git a/cloud_storage_handler/api/elixircloud/csh/models.py b/cloud_storage_handler/api/elixircloud/csh/models.py index 2768b10..2008fbc 100644 --- a/cloud_storage_handler/api/elixircloud/csh/models.py +++ b/cloud_storage_handler/api/elixircloud/csh/models.py @@ -14,7 +14,11 @@ class MinioConfig(BaseModel): example="localhost", ) port: int = Field( - ..., description="The port on which the MinIO server is running.", example=9000 + ..., + description="The port on which the MinIO server is running.", + example=9000, + ge=1, + le=65535, ) access_key: str = Field( ..., diff --git a/cloud_storage_handler/consts.py b/cloud_storage_handler/consts.py index 5f5da72..f748f11 100644 --- a/cloud_storage_handler/consts.py +++ b/cloud_storage_handler/consts.py @@ -1,27 +1,46 @@ """This module handles the loading of environment variables for MinIO configuration.""" +from typing import Dict + +from minio import Minio + from cloud_storage_handler.minio_client import MinioClient -def GetMinioConfiguration(): # type: ignore - """Get minio configuration.""" - minio_config = MinioClient() - return minio_config.response() +def load_minio_config() -> Dict[str, any]: + """Load MinIO configuration. + + Returns: + Dict[str, any]: A dictionary containing MinIO configuration details. + """ + minio_client = MinioClient() + return minio_client.get_minio_config() + + +def initialize_minio_client(config: Dict[str, any]) -> Minio: + """Initialize the MinIO client with the given configuration. + + Args: + config (Dict[str, any]): A dictionary containing configuration details. + Returns: + Minio: A MinIO client instance. + """ + endpoint = f"{config['hostname']}:{config['port']}" + return Minio( + endpoint, + access_key=config["access_key"], + secret_key=config["secret_key"], + secure=config["is_secure"], + ) -minio_config_data = GetMinioConfiguration() -MINIO_HOSTNAME = minio_config_data["hostname"] -MINIO_PORT = minio_config_data["port"] -MINIO_ENDPOINT = f"{MINIO_HOSTNAME}:{MINIO_PORT}" -MINIO_SECURE = minio_config_data["is_secure"] +def main(): + """Main function to load configuration and initialize the MinIO client.""" + minio_client = MinioClient() + minio_client.create_bucket(minio_client.bucket_name) + print(minio_client.client) + return minio_client.client -minio_client_instance = MinioClient() -minio_client = minio_client_instance.initialise_minio( - endpoint=MINIO_ENDPOINT, - access_key=minio_config_data["access_key"], - secret_key=minio_config_data["secret_key"], - secure=MINIO_SECURE, -) -minio_client_instance.create_bucket(minio_config_data["bucket_name"]) +minio_client = main() diff --git a/cloud_storage_handler/minio_client.py b/cloud_storage_handler/minio_client.py index aab32e5..ced941f 100644 --- a/cloud_storage_handler/minio_client.py +++ b/cloud_storage_handler/minio_client.py @@ -11,32 +11,29 @@ logger = logging.getLogger(__name__) -class MinioClient: - """A class to manage MinIO client configuration and bucket creation.""" - - def __init__(self) -> None: - """Initializes the MinIO client class.""" - self.minio_config_data = None - self.config = Foca(config_file=get_config_path()).conf - self.client = None # MinIO client will be initialized later - - def initialise_minio(self, endpoint, access_key, secret_key, secure): - """Initialize the MinIO client with provided configurations.""" - self.client = Minio( - endpoint=endpoint, - access_key=access_key, - secret_key=secret_key, - secure=secure, - ) - return self.client - - def create_bucket(self, bucket_name): - """Create a new bucket if it does not already exist.""" - if self.client is None: - raise RuntimeError("MinIO client is not initialized.") - - if not self.client.bucket_exists(bucket_name): - self.client.make_bucket(bucket_name) +class MinioConfig: + """Handles the loading and parsing of MinIO configuration files.""" + + def __init__(self, config_file=None): + """Initialize MinioConfig with a configuration file. + + Args: + config_file (str): Path to the configuration file. If not provided, + uses the default path. + """ + config_file = config_file or get_config_path() + self.config = Foca(config_file=config_file).conf + + def get_minio_config(self): + """Retrieve the MinIO configuration. + + Returns: + dict: A dictionary containing the MinIO configuration. + """ + try: + return self.get_minio_from_config() + except ConfigNotFoundError: + return self.get_default_minio_config() def get_default_minio_config(self): """Get the default minio configuration.""" @@ -59,16 +56,41 @@ def get_minio_from_config(self): minio_config_data = self.config.custom.get("minio") if not minio_config_data: - raise ConfigNotFoundError("Service info not found in custom configuration.") + raise ConfigNotFoundError( + "MinIO configuration not found in custom configuration." + ) return minio_config_data - def response(self): - """Returns minio configuration response.""" - if self.minio_config_data is None: - try: - self.minio_config_data = self.get_minio_from_config() - except ConfigNotFoundError: - self.minio_config_data = self.get_default_minio_config() - return self.minio_config_data +class MinioClient: + """A class to manage MinIO client configuration and bucket creation.""" + + def __init__(self): + """Initialize the MinIO client and create bucket if necessary.""" + config = MinioConfig().get_minio_config() + self.client = self.initialise_minio( + endpoint=f"{config['hostname']}:{config['port']}", + access_key=config["access_key"], + secret_key=config["secret_key"], + secure=config["is_secure"], + ) + self.bucket_name = config["bucket_name"] + + def initialise_minio(self, endpoint, access_key, secret_key, secure): + """Initialize the MinIO client with provided configurations.""" + self.client = Minio( + endpoint=endpoint, + access_key=access_key, + secret_key=secret_key, + secure=secure, + ) + return self.client + + def create_bucket(self, bucket_name): + """Create a new bucket if it does not already exist.""" + if self.client is None: + raise RuntimeError("MinIO client is not initialized.") + + if not self.client.bucket_exists(bucket_name): + self.client.make_bucket(bucket_name) From cbcf33021a3ca0c1a0d8abc69bfd3e635f6ac7e4 Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Thu, 5 Sep 2024 23:36:35 +0530 Subject: [PATCH 04/13] fixed type issue Signed-off-by: Pratiksha Sankhe --- cloud_storage_handler/consts.py | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/cloud_storage_handler/consts.py b/cloud_storage_handler/consts.py index f748f11..e97b1b6 100644 --- a/cloud_storage_handler/consts.py +++ b/cloud_storage_handler/consts.py @@ -1,27 +1,13 @@ -"""This module handles the loading of environment variables for MinIO configuration.""" - -from typing import Dict - -from minio import Minio +from typing import Any, Dict from cloud_storage_handler.minio_client import MinioClient +from minio import Minio - -def load_minio_config() -> Dict[str, any]: - """Load MinIO configuration. - - Returns: - Dict[str, any]: A dictionary containing MinIO configuration details. - """ - minio_client = MinioClient() - return minio_client.get_minio_config() - - -def initialize_minio_client(config: Dict[str, any]) -> Minio: +def initialize_minio_client(config: Dict[str, Any]) -> Minio: """Initialize the MinIO client with the given configuration. Args: - config (Dict[str, any]): A dictionary containing configuration details. + config (Dict[str, Any]): A dictionary containing configuration details. Returns: Minio: A MinIO client instance. @@ -34,7 +20,6 @@ def initialize_minio_client(config: Dict[str, any]) -> Minio: secure=config["is_secure"], ) - def main(): """Main function to load configuration and initialize the MinIO client.""" minio_client = MinioClient() @@ -42,5 +27,4 @@ def main(): print(minio_client.client) return minio_client.client - -minio_client = main() +minio_client = main() \ No newline at end of file From 58070c9ada309916979263f471bb3d541288f53f Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Thu, 5 Sep 2024 23:40:18 +0530 Subject: [PATCH 05/13] fixed lint issue Signed-off-by: Pratiksha Sankhe --- cloud_storage_handler/consts.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cloud_storage_handler/consts.py b/cloud_storage_handler/consts.py index e97b1b6..34316cd 100644 --- a/cloud_storage_handler/consts.py +++ b/cloud_storage_handler/consts.py @@ -1,8 +1,16 @@ +"""This module handles MinIO configuration and client initialization. + +It provides functions to load MinIO configuration, initialize the MinIO client +with the configuration details, and perform operations such as bucket creation. +""" + from typing import Any, Dict -from cloud_storage_handler.minio_client import MinioClient from minio import Minio +from cloud_storage_handler.minio_client import MinioClient + + def initialize_minio_client(config: Dict[str, Any]) -> Minio: """Initialize the MinIO client with the given configuration. @@ -20,6 +28,7 @@ def initialize_minio_client(config: Dict[str, Any]) -> Minio: secure=config["is_secure"], ) + def main(): """Main function to load configuration and initialize the MinIO client.""" minio_client = MinioClient() @@ -27,4 +36,5 @@ def main(): print(minio_client.client) return minio_client.client -minio_client = main() \ No newline at end of file + +minio_client = main() From 8897c4a4e9ff25ebfd2beb28b0cf0830753431bd Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Wed, 11 Sep 2024 00:04:00 +0530 Subject: [PATCH 06/13] modified files Signed-off-by: Pratiksha Sankhe --- .../api/elixircloud/csh/models.py | 65 +++++++++---------- .../{ => clients}/minio_client.py | 16 +++-- cloud_storage_handler/consts.py | 40 ------------ cloud_storage_handler/main.py | 5 ++ deployment/config.yaml | 3 +- 5 files changed, 44 insertions(+), 85 deletions(-) rename cloud_storage_handler/{ => clients}/minio_client.py (87%) delete mode 100644 cloud_storage_handler/consts.py diff --git a/cloud_storage_handler/api/elixircloud/csh/models.py b/cloud_storage_handler/api/elixircloud/csh/models.py index 2008fbc..945b06b 100644 --- a/cloud_storage_handler/api/elixircloud/csh/models.py +++ b/cloud_storage_handler/api/elixircloud/csh/models.py @@ -2,43 +2,36 @@ from typing import Annotated -from pydantic import BaseModel, Field, constr +from pydantic import BaseModel, conint, constr class MinioConfig(BaseModel): - """Configuration for MinIO.""" + """Configuration for MinIO. - hostname: str = Field( - ..., - description="The hostname where the MinIO server is running.", - example="localhost", - ) - port: int = Field( - ..., - description="The port on which the MinIO server is running.", - example=9000, - ge=1, - le=65535, - ) - access_key: str = Field( - ..., - description="The access key used for authentication with MinIO.", - example="minioadmin", - ) - secret_key: str = Field( - ..., - description="The secret key used for authentication with MinIO.", - example="minioadmin", - ) - is_secure: bool = Field( - ..., - description=( - "Specifies whether the connection to MinIO should be secure (HTTPS)." - ), - example=False, - ) - bucket_name: Annotated[str, constr(min_length=1)] = Field( - ..., - description="The name of the bucket where files are stored.", - example="files", - ) + Attributes: + hostname (str): The hostname where the MinIO server is running. + Defaults to 'localhost'. + port (int): The port on which the MinIO server is running. + Must be between 1 and 65535. Defaults to 9000. + access_key (str): The access key used for authentication with MinIO. + Defaults to 'minioadmin'. + secret_key (str): The secret key used for authentication with MinIO. + Defaults to 'minioadmin'. + bucket_name (str): The name of the bucket where files are stored. + Must be at least 1 character long. Defaults to 'files'. + + Examples: + MinioConfig( + hostname="localhost", + port=9000, + access_key="minioadmin", + secret_key="minioadmin", + bucket_name="files" + ) + """ + + hostname: str = "localhost" + port: Annotated[int, conint(ge=1, le=65535)] = 9000 + access_key: str = "minioadmin" + secret_key: str = "minioadmin" + bucket_name: Annotated[str, constr(min_length=1)] = "files" diff --git a/cloud_storage_handler/minio_client.py b/cloud_storage_handler/clients/minio_client.py similarity index 87% rename from cloud_storage_handler/minio_client.py rename to cloud_storage_handler/clients/minio_client.py index ced941f..c374cd4 100644 --- a/cloud_storage_handler/minio_client.py +++ b/cloud_storage_handler/clients/minio_client.py @@ -45,7 +45,6 @@ def get_default_minio_config(self): "port": 9000, "access_key": "minioadmin", "secret_key": "minioadmin", - "is_secure": False, "bucket_name": "files", } @@ -73,7 +72,7 @@ def __init__(self): endpoint=f"{config['hostname']}:{config['port']}", access_key=config["access_key"], secret_key=config["secret_key"], - secure=config["is_secure"], + secure=False, ) self.bucket_name = config["bucket_name"] @@ -83,14 +82,17 @@ def initialise_minio(self, endpoint, access_key, secret_key, secure): endpoint=endpoint, access_key=access_key, secret_key=secret_key, - secure=secure, + secure=False, ) return self.client - def create_bucket(self, bucket_name): - """Create a new bucket if it does not already exist.""" + def create_bucket(self): + """Creation of bucket using the configured bucket name.""" if self.client is None: raise RuntimeError("MinIO client is not initialized.") - if not self.client.bucket_exists(bucket_name): - self.client.make_bucket(bucket_name) + if not self.client.bucket_exists(self.bucket_name): + self.client.make_bucket(self.bucket_name) + logger.info(f"Bucket '{self.bucket_name}' created.") + else: + logger.info(f"Bucket '{self.bucket_name}' already exists.") diff --git a/cloud_storage_handler/consts.py b/cloud_storage_handler/consts.py deleted file mode 100644 index 34316cd..0000000 --- a/cloud_storage_handler/consts.py +++ /dev/null @@ -1,40 +0,0 @@ -"""This module handles MinIO configuration and client initialization. - -It provides functions to load MinIO configuration, initialize the MinIO client -with the configuration details, and perform operations such as bucket creation. -""" - -from typing import Any, Dict - -from minio import Minio - -from cloud_storage_handler.minio_client import MinioClient - - -def initialize_minio_client(config: Dict[str, Any]) -> Minio: - """Initialize the MinIO client with the given configuration. - - Args: - config (Dict[str, Any]): A dictionary containing configuration details. - - Returns: - Minio: A MinIO client instance. - """ - endpoint = f"{config['hostname']}:{config['port']}" - return Minio( - endpoint, - access_key=config["access_key"], - secret_key=config["secret_key"], - secure=config["is_secure"], - ) - - -def main(): - """Main function to load configuration and initialize the MinIO client.""" - minio_client = MinioClient() - minio_client.create_bucket(minio_client.bucket_name) - print(minio_client.client) - return minio_client.client - - -minio_client = main() diff --git a/cloud_storage_handler/main.py b/cloud_storage_handler/main.py index 4f8fcf0..126d349 100644 --- a/cloud_storage_handler/main.py +++ b/cloud_storage_handler/main.py @@ -7,8 +7,13 @@ from connexion import FlaskApp from foca import Foca +from cloud_storage_handler.clients.minio_client import MinioClient + logger = logging.getLogger(__name__) +minio_client = MinioClient() +minio_client.create_bucket() + def init_app() -> FlaskApp: """Initialize and return the FOCA app. diff --git a/deployment/config.yaml b/deployment/config.yaml index f322f77..ee0aa70 100644 --- a/deployment/config.yaml +++ b/deployment/config.yaml @@ -39,10 +39,9 @@ custom: # MinIO configuration based on MinIO model minio: hostname: localhost - port: 9001 + port: 9000 access_key: minioadmin secret_key: minioadmin - is_secure: false bucket_name: 'files' exceptions: From 9595d74baaf52911330022b67eb1f4b883f99518 Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Wed, 11 Sep 2024 00:05:09 +0530 Subject: [PATCH 07/13] docs Signed-off-by: Pratiksha Sankhe --- docs/source/pages/cloud_storage_handler.rst | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/docs/source/pages/cloud_storage_handler.rst b/docs/source/pages/cloud_storage_handler.rst index 156c83c..15635d9 100644 --- a/docs/source/pages/cloud_storage_handler.rst +++ b/docs/source/pages/cloud_storage_handler.rst @@ -12,14 +12,6 @@ Subpackages Submodules ---------- -cloud\_storage\_handler.consts module -------------------------------------- - -.. automodule:: cloud_storage_handler.consts - :members: - :undoc-members: - :show-inheritance: - cloud\_storage\_handler.custom\_config module --------------------------------------------- @@ -44,14 +36,6 @@ cloud\_storage\_handler.main module :undoc-members: :show-inheritance: -cloud\_storage\_handler.minio\_client module --------------------------------------------- - -.. automodule:: cloud_storage_handler.minio_client - :members: - :undoc-members: - :show-inheritance: - cloud\_storage\_handler.utils module ------------------------------------ From 32b1849e1ad2d2746c52b1533efdf5018dacd1df Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Wed, 11 Sep 2024 22:37:52 +0530 Subject: [PATCH 08/13] updated doc Signed-off-by: Pratiksha Sankhe --- cloud_storage_handler/clients/minio.py | 6 ++++++ cloud_storage_handler/main.py | 7 ++----- 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 cloud_storage_handler/clients/minio.py diff --git a/cloud_storage_handler/clients/minio.py b/cloud_storage_handler/clients/minio.py new file mode 100644 index 0000000..a1352f6 --- /dev/null +++ b/cloud_storage_handler/clients/minio.py @@ -0,0 +1,6 @@ +from cloud_storage_handler.clients.minio_client import MinioClient + +def get_minio_client(): + client = MinioClient() + client.create_bucket() + return client \ No newline at end of file diff --git a/cloud_storage_handler/main.py b/cloud_storage_handler/main.py index 126d349..d390f8e 100644 --- a/cloud_storage_handler/main.py +++ b/cloud_storage_handler/main.py @@ -7,14 +7,10 @@ from connexion import FlaskApp from foca import Foca -from cloud_storage_handler.clients.minio_client import MinioClient +from cloud_storage_handler.clients.minio import get_minio_client logger = logging.getLogger(__name__) -minio_client = MinioClient() -minio_client.create_bucket() - - def init_app() -> FlaskApp: """Initialize and return the FOCA app. @@ -52,6 +48,7 @@ def init_app() -> FlaskApp: def main() -> None: """Run FOCA application.""" app = init_app() + minio_client = get_minio_client() app.run(port=app.port) From 6ae493ad61c141f059b8189e7de77ad86ec7718a Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Wed, 11 Sep 2024 22:42:06 +0530 Subject: [PATCH 09/13] fixed workflows Signed-off-by: Pratiksha Sankhe --- cloud_storage_handler/clients/minio.py | 14 +++++++++++++- cloud_storage_handler/main.py | 3 ++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cloud_storage_handler/clients/minio.py b/cloud_storage_handler/clients/minio.py index a1352f6..d09477f 100644 --- a/cloud_storage_handler/clients/minio.py +++ b/cloud_storage_handler/clients/minio.py @@ -1,6 +1,18 @@ +"""Minio client module. + +This module provides functionality to interact with the Minio cloud storage +service. +""" + from cloud_storage_handler.clients.minio_client import MinioClient + def get_minio_client(): + """Get a Minio client and create a bucket. + + Returns: + MinioClient: An initialized Minio client with a created bucket. + """ client = MinioClient() client.create_bucket() - return client \ No newline at end of file + return client diff --git a/cloud_storage_handler/main.py b/cloud_storage_handler/main.py index d390f8e..f0befd0 100644 --- a/cloud_storage_handler/main.py +++ b/cloud_storage_handler/main.py @@ -11,6 +11,7 @@ logger = logging.getLogger(__name__) + def init_app() -> FlaskApp: """Initialize and return the FOCA app. @@ -48,7 +49,7 @@ def init_app() -> FlaskApp: def main() -> None: """Run FOCA application.""" app = init_app() - minio_client = get_minio_client() + get_minio_client() app.run(port=app.port) From f1736e5813730ae123115919e7f1e3f4ef841c74 Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Wed, 11 Sep 2024 23:04:28 +0530 Subject: [PATCH 10/13] changed some structure Signed-off-by: Pratiksha Sankhe --- cloud_storage_handler/clients/minio.py | 5 +++-- cloud_storage_handler/clients/minio_client.py | 19 ++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/cloud_storage_handler/clients/minio.py b/cloud_storage_handler/clients/minio.py index d09477f..3c97cec 100644 --- a/cloud_storage_handler/clients/minio.py +++ b/cloud_storage_handler/clients/minio.py @@ -13,6 +13,7 @@ def get_minio_client(): Returns: MinioClient: An initialized Minio client with a created bucket. """ - client = MinioClient() - client.create_bucket() + init_client=MinioClient() + client=init_client.get_client() + init_client.create_bucket() return client diff --git a/cloud_storage_handler/clients/minio_client.py b/cloud_storage_handler/clients/minio_client.py index c374cd4..6c52231 100644 --- a/cloud_storage_handler/clients/minio_client.py +++ b/cloud_storage_handler/clients/minio_client.py @@ -68,23 +68,16 @@ class MinioClient: def __init__(self): """Initialize the MinIO client and create bucket if necessary.""" config = MinioConfig().get_minio_config() - self.client = self.initialise_minio( + self.client = Minio( endpoint=f"{config['hostname']}:{config['port']}", access_key=config["access_key"], secret_key=config["secret_key"], - secure=False, + secure=False ) self.bucket_name = config["bucket_name"] - def initialise_minio(self, endpoint, access_key, secret_key, secure): - """Initialize the MinIO client with provided configurations.""" - self.client = Minio( - endpoint=endpoint, - access_key=access_key, - secret_key=secret_key, - secure=False, - ) - return self.client + # Create bucket if it doesn't exist + self.create_bucket() def create_bucket(self): """Creation of bucket using the configured bucket name.""" @@ -96,3 +89,7 @@ def create_bucket(self): logger.info(f"Bucket '{self.bucket_name}' created.") else: logger.info(f"Bucket '{self.bucket_name}' already exists.") + + def get_client(self): + """Return the MinIO client.""" + return self.client From 338af10e2bc6bf912474952f82a40b9f78a49c0b Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Wed, 11 Sep 2024 23:06:28 +0530 Subject: [PATCH 11/13] fixed workflow Signed-off-by: Pratiksha Sankhe --- cloud_storage_handler/clients/minio.py | 4 ++-- cloud_storage_handler/clients/minio_client.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cloud_storage_handler/clients/minio.py b/cloud_storage_handler/clients/minio.py index 3c97cec..c5f18b8 100644 --- a/cloud_storage_handler/clients/minio.py +++ b/cloud_storage_handler/clients/minio.py @@ -13,7 +13,7 @@ def get_minio_client(): Returns: MinioClient: An initialized Minio client with a created bucket. """ - init_client=MinioClient() - client=init_client.get_client() + init_client = MinioClient() + client = init_client.get_client() init_client.create_bucket() return client diff --git a/cloud_storage_handler/clients/minio_client.py b/cloud_storage_handler/clients/minio_client.py index 6c52231..6f136fa 100644 --- a/cloud_storage_handler/clients/minio_client.py +++ b/cloud_storage_handler/clients/minio_client.py @@ -72,7 +72,7 @@ def __init__(self): endpoint=f"{config['hostname']}:{config['port']}", access_key=config["access_key"], secret_key=config["secret_key"], - secure=False + secure=False, ) self.bucket_name = config["bucket_name"] From fee822890c8c3cc720c25ea6fac8af67f9285e34 Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Fri, 20 Sep 2024 21:58:07 +0530 Subject: [PATCH 12/13] made changes Signed-off-by: Pratiksha Sankhe --- .../api/elixircloud/csh/constants.py | 48 ++++++++++++++++++ .../api/elixircloud/csh/models.py | 10 ++-- cloud_storage_handler/clients/minio.py | 3 ++ cloud_storage_handler/clients/minio_client.py | 50 ++++++++++--------- cloud_storage_handler/main.py | 4 +- ...ud_storage_handler.api.elixircloud.csh.rst | 8 +++ 6 files changed, 92 insertions(+), 31 deletions(-) create mode 100644 cloud_storage_handler/api/elixircloud/csh/constants.py diff --git a/cloud_storage_handler/api/elixircloud/csh/constants.py b/cloud_storage_handler/api/elixircloud/csh/constants.py new file mode 100644 index 0000000..e9835ba --- /dev/null +++ b/cloud_storage_handler/api/elixircloud/csh/constants.py @@ -0,0 +1,48 @@ +"""Constants.""" + +from pydantic import BaseModel + +from cloud_storage_handler.api.elixircloud.csh.models import MinioConfig + + +class MinioConstants(BaseModel): + """All the constants related to MinIO for CSH.""" + + hostname: str = "localhost" + port: int = 9000 + access_key: str = "minioadmin" + secret_key: str = "minioadmin" + bucket_name: str = "files" + + class Config: + """Configuration for MinioConstants class.""" + + frozen = True + + +class CshConstants(BaseModel): + """All the constants related to CSH and related services. + + Attributes: + foca_config_path: Path to FOCA configuration for CSH. + minio_constants: Constants related to MinIO configuration. + """ + + foca_config_path: str = "CSH_FOCA_CONFIG_PATH" + minio_constants: MinioConstants = MinioConstants() + + class Config: + """Configuration for CshConstants class.""" + + frozen = True + + def get_minio_config(self) -> MinioConfig: + """Return MinIO configuration as a Pydantic model.""" + minio_config = self.minio_constants + return MinioConfig( + hostname=minio_config.hostname, + port=minio_config.port, + access_key=minio_config.access_key, + secret_key=minio_config.secret_key, + bucket_name=minio_config.bucket_name, + ) diff --git a/cloud_storage_handler/api/elixircloud/csh/models.py b/cloud_storage_handler/api/elixircloud/csh/models.py index 945b06b..3a5c803 100644 --- a/cloud_storage_handler/api/elixircloud/csh/models.py +++ b/cloud_storage_handler/api/elixircloud/csh/models.py @@ -9,15 +9,15 @@ class MinioConfig(BaseModel): """Configuration for MinIO. Attributes: - hostname (str): The hostname where the MinIO server is running. + hostname : The hostname where the MinIO server is running. Defaults to 'localhost'. - port (int): The port on which the MinIO server is running. + port : The port on which the MinIO server is running. Must be between 1 and 65535. Defaults to 9000. - access_key (str): The access key used for authentication with MinIO. + access_key : The access key used for authentication with MinIO. Defaults to 'minioadmin'. - secret_key (str): The secret key used for authentication with MinIO. + secret_key : The secret key used for authentication with MinIO. Defaults to 'minioadmin'. - bucket_name (str): The name of the bucket where files are stored. + bucket_name : The name of the bucket where files are stored. Must be at least 1 character long. Defaults to 'files'. Examples: diff --git a/cloud_storage_handler/clients/minio.py b/cloud_storage_handler/clients/minio.py index c5f18b8..1d920de 100644 --- a/cloud_storage_handler/clients/minio.py +++ b/cloud_storage_handler/clients/minio.py @@ -17,3 +17,6 @@ def get_minio_client(): client = init_client.get_client() init_client.create_bucket() return client + + +minio_client = get_minio_client() diff --git a/cloud_storage_handler/clients/minio_client.py b/cloud_storage_handler/clients/minio_client.py index 6f136fa..662b957 100644 --- a/cloud_storage_handler/clients/minio_client.py +++ b/cloud_storage_handler/clients/minio_client.py @@ -5,20 +5,22 @@ from foca import Foca from minio import Minio +from cloud_storage_handler.api.elixircloud.csh.constants import CshConstants +from cloud_storage_handler.api.elixircloud.csh.models import MinioConfig from cloud_storage_handler.exceptions import ConfigNotFoundError from cloud_storage_handler.utils import get_config_path logger = logging.getLogger(__name__) -class MinioConfig: +class MinioConfigurationData: """Handles the loading and parsing of MinIO configuration files.""" def __init__(self, config_file=None): """Initialize MinioConfig with a configuration file. Args: - config_file (str): Path to the configuration file. If not provided, + config_file : Path to the configuration file. If not provided, uses the default path. """ config_file = config_file or get_config_path() @@ -28,38 +30,40 @@ def get_minio_config(self): """Retrieve the MinIO configuration. Returns: - dict: A dictionary containing the MinIO configuration. + A dictionary containing the MinIO configuration. """ try: return self.get_minio_from_config() except ConfigNotFoundError: return self.get_default_minio_config() - def get_default_minio_config(self): + def get_default_minio_config(self) -> MinioConfig: """Get the default minio configuration.""" logger.warning( "Minio configuration not found in config. Using default configuration." ) - return { - "hostname": "localhost", - "port": 9000, - "access_key": "minioadmin", - "secret_key": "minioadmin", - "bucket_name": "files", - } - - def get_minio_from_config(self): + csh_constants = CshConstants() + minio_config = csh_constants.get_minio_config() + return minio_config + + def get_minio_from_config(self) -> MinioConfig: """Returns minio configuration from config.""" if not self.config.custom: raise ConfigNotFoundError("Custom configuration not found.") - minio_config_data = self.config.custom.get("minio") - if not minio_config_data: + minio_config = self.config.custom.get("minio") + if not minio_config: raise ConfigNotFoundError( "MinIO configuration not found in custom configuration." ) - return minio_config_data + return MinioConfig( + hostname=minio_config["hostname"], + port=minio_config["port"], + access_key=minio_config["access_key"], + secret_key=minio_config["secret_key"], + bucket_name=minio_config["bucket_name"], + ) class MinioClient: @@ -67,23 +71,21 @@ class MinioClient: def __init__(self): """Initialize the MinIO client and create bucket if necessary.""" - config = MinioConfig().get_minio_config() + config = MinioConfigurationData().get_minio_config() + print(config) self.client = Minio( - endpoint=f"{config['hostname']}:{config['port']}", - access_key=config["access_key"], - secret_key=config["secret_key"], + endpoint=f"{config.hostname}:{config.port}", + access_key=config.access_key, + secret_key=config.secret_key, secure=False, ) - self.bucket_name = config["bucket_name"] + self.bucket_name = config.bucket_name # Create bucket if it doesn't exist self.create_bucket() def create_bucket(self): """Creation of bucket using the configured bucket name.""" - if self.client is None: - raise RuntimeError("MinIO client is not initialized.") - if not self.client.bucket_exists(self.bucket_name): self.client.make_bucket(self.bucket_name) logger.info(f"Bucket '{self.bucket_name}' created.") diff --git a/cloud_storage_handler/main.py b/cloud_storage_handler/main.py index f0befd0..cb11463 100644 --- a/cloud_storage_handler/main.py +++ b/cloud_storage_handler/main.py @@ -7,7 +7,7 @@ from connexion import FlaskApp from foca import Foca -from cloud_storage_handler.clients.minio import get_minio_client +from cloud_storage_handler.clients.minio import minio_client logger = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def init_app() -> FlaskApp: def main() -> None: """Run FOCA application.""" app = init_app() - get_minio_client() + _ = minio_client app.run(port=app.port) diff --git a/docs/source/pages/cloud_storage_handler.api.elixircloud.csh.rst b/docs/source/pages/cloud_storage_handler.api.elixircloud.csh.rst index 8cc012a..024a8a0 100644 --- a/docs/source/pages/cloud_storage_handler.api.elixircloud.csh.rst +++ b/docs/source/pages/cloud_storage_handler.api.elixircloud.csh.rst @@ -4,6 +4,14 @@ cloud\_storage\_handler.api.elixircloud.csh package Submodules ---------- +cloud\_storage\_handler.api.elixircloud.csh.constants module +------------------------------------------------------------ + +.. automodule:: cloud_storage_handler.api.elixircloud.csh.constants + :members: + :undoc-members: + :show-inheritance: + cloud\_storage\_handler.api.elixircloud.csh.controllers module -------------------------------------------------------------- From bfac0b9b4a83813d8c95aa63ae4b8c80b058b16e Mon Sep 17 00:00:00 2001 From: Pratiksha Sankhe Date: Mon, 23 Sep 2024 20:21:10 +0530 Subject: [PATCH 13/13] added return type Signed-off-by: Pratiksha Sankhe --- cloud_storage_handler/clients/minio_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud_storage_handler/clients/minio_client.py b/cloud_storage_handler/clients/minio_client.py index 662b957..e3a5e8a 100644 --- a/cloud_storage_handler/clients/minio_client.py +++ b/cloud_storage_handler/clients/minio_client.py @@ -26,7 +26,7 @@ def __init__(self, config_file=None): config_file = config_file or get_config_path() self.config = Foca(config_file=config_file).conf - def get_minio_config(self): + def get_minio_config(self) -> MinioConfig: """Retrieve the MinIO configuration. Returns: