diff --git a/runners/s3-benchrunner-python/README.md b/runners/s3-benchrunner-python/README.md index 7d9175ef..48125b6c 100644 --- a/runners/s3-benchrunner-python/README.md +++ b/runners/s3-benchrunner-python/README.md @@ -80,7 +80,7 @@ python3 -m pip install --upgrade "boto3[crt]" ## Building locally -To test against in-development of these libraries, we'll install from source: +To test against in-development of these libraries, install from source: First, create a virtual environment, to isolate your dev versions from system defaults: ```sh @@ -89,12 +89,12 @@ source .venv/bin/activate ``` Now make a build dir somewhere. -We're going to pull the source code for dependencies and install them... +You're going to pull the source code for dependencies and install them... ``` (.venv) cd path/to/my/build/dir ``` -First, AWS CLI (`--editable` so we can modify its source without reinstalling): +First, AWS CLI (`--editable` so you can modify its source without reinstalling): ```sh (.venv) git clone --branch v2 https://github.com/aws/aws-cli.git (.venv) python3 -m pip install --editable aws-cli @@ -124,7 +124,7 @@ Finally aws-crt-python: (.venv) python3 -m pip install --editable aws-crt-python ``` pip complains that the newly installed 1.0.0.dev0 clashes -with the version requirements from other packages, but we ignore this. +with the version requirements from other packages, but you can ignore this. Now, you can execute the runner using your virtual environment with all the latest code: ```sh diff --git a/runners/s3-benchrunner-python/scripts/build.py b/runners/s3-benchrunner-python/scripts/build.py index d6e621ec..cfa47bc7 100755 --- a/runners/s3-benchrunner-python/scripts/build.py +++ b/runners/s3-benchrunner-python/scripts/build.py @@ -33,31 +33,16 @@ def fetch_git_repo(url: str, dir: Path, main_branch: str, preferred_branch: Opti run(fetch_cmd) -def build_cli(work_dir: Path, branch: Optional[str], venv_python: str): - cli_dir = work_dir.joinpath('aws-cli') - - # fetch git repo (if necessary) - fetch_git_repo('https://github.com/aws/aws-cli.git', cli_dir, - main_branch='v2', preferred_branch=branch) - - # install CLI into virtual env - # use --editable so we don't need to reinstall after simple file edits - run([venv_python, '-m', 'pip', 'install', '--editable', str(cli_dir)]) - - -def build_crt(work_dir: Path, branch: Optional[str], venv_python: str): - crt_dir = work_dir.joinpath('aws-crt-python') - - # fetch git repo (if necessary) - fetch_git_repo('https://github.com/awslabs/aws-crt-python.git', crt_dir, - main_branch='main', preferred_branch=branch) - - # for faster C compilation - os.environ['CMAKE_BUILD_PARALLEL_LEVEL'] = str(os.cpu_count()) +def fetch_and_install(url: str, + dir: Path, + main_branch: str, + preferred_branch: Optional[str], + venv_python: str): + fetch_git_repo(url, dir, main_branch, preferred_branch) # install into virtual env # use --editable so we don't need to reinstall after simple file edits - run([venv_python, '-m', 'pip', 'install', '--editable', str(crt_dir)]) + run([venv_python, '-m', 'pip', 'install', '--editable', str(dir)]) if __name__ == '__main__': @@ -74,16 +59,45 @@ def build_crt(work_dir: Path, branch: Optional[str], venv_python: str): # upgrade pip to avoid warnings run([venv_python, '-m', 'pip', 'install', '--upgrade', 'pip']) - # install aws-cli from Github - build_cli(work_dir, args.branch, venv_python) - - # the runner uses boto3 too - run([venv_python, '-m', 'pip', 'install', 'boto3']) - - # install aws-crt-python from Github - # (pip complains that the newly installed 1.0.0.dev0 clashes - # with the version requirements from awscli, but we ignore this) - build_crt(work_dir, args.branch, venv_python) + fetch_and_install( + url='https://github.com/aws/aws-cli.git', + dir=work_dir.joinpath('aws-cli'), + main_branch='v2', + preferred_branch=args.branch, + venv_python=venv_python) + + fetch_and_install( + url='https://github.com/boto/boto3.git', + dir=work_dir.joinpath('boto3'), + main_branch='develop', + preferred_branch=args.branch, + venv_python=venv_python) + + fetch_and_install( + url='https://github.com/boto/s3transfer.git', + dir=work_dir.joinpath('s3transfer'), + main_branch='develop', + preferred_branch=args.branch, + venv_python=venv_python) + + fetch_and_install( + url='https://github.com/boto/botocore.git', + dir=work_dir.joinpath('botocore'), + main_branch='develop', + preferred_branch=args.branch, + venv_python=venv_python) + + # install aws-crt-python + # set CMAKE_BUILD_PARALLEL_LEVEL for faster C build + # NOTE: (pip complains that the newly installed 1.0.0.dev0 clashes + # with the version requirements from other packages, but we ignore this) + os.environ['CMAKE_BUILD_PARALLEL_LEVEL'] = str(os.cpu_count()) + fetch_and_install( + url='https://github.com/awslabs/aws-crt-python.git', + dir=work_dir.joinpath('aws-crt-python'), + main_branch='main', + preferred_branch=args.branch, + venv_python=venv_python) runner_dir = Path(__file__).parent.parent.resolve() # normalize path runner_py = str(runner_dir.joinpath('benchrunner.py'))