From a473549db19b14ca3430d22f5e0bf731db52daa9 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Sat, 7 Oct 2023 17:27:50 +0000 Subject: [PATCH] flush stdout before running subprocess, so things print in order --- runners/s3-benchrunner-c/scripts/build.py | 2 +- runners/s3-benchrunner-cli/benchrunner.py | 7 +++--- runners/s3-benchrunner-cli/scripts/build.py | 2 +- .../s3-benchrunner-crt-java/scripts/build.py | 2 +- .../s3-benchrunner-crt-java/scripts/lint.py | 2 +- runners/s3-benchrunner-python/benchrunner.py | 21 ++++++------------ scripts/fetch-git-repo.py | 4 ++-- scripts/install-tools-AL2023.py | 2 +- scripts/lint-python.py | 2 +- scripts/run-benchmarks.py | 22 ++++++++++++++----- 10 files changed, 36 insertions(+), 30 deletions(-) diff --git a/runners/s3-benchrunner-c/scripts/build.py b/runners/s3-benchrunner-c/scripts/build.py index 676b5b7c..a91fe724 100755 --- a/runners/s3-benchrunner-c/scripts/build.py +++ b/runners/s3-benchrunner-c/scripts/build.py @@ -31,7 +31,7 @@ def run(cmd_args: list[str]): - print(f'> {subprocess.list2cmdline(cmd_args)}') + print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True) subprocess.run(cmd_args, check=True) diff --git a/runners/s3-benchrunner-cli/benchrunner.py b/runners/s3-benchrunner-cli/benchrunner.py index 004755c3..5e98f5ff 100755 --- a/runners/s3-benchrunner-cli/benchrunner.py +++ b/runners/s3-benchrunner-cli/benchrunner.py @@ -234,7 +234,7 @@ def _using_all_files_in_dir(self, action: str, prefix: str) -> bool: paginator = s3.get_paginator('list_objects_v2') for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix): for obj in page['Contents']: - if obj['Key'] in all_task_keys: + if not obj['Key'] in all_task_keys: return False else: # upload @@ -252,7 +252,7 @@ def run(self): 'input': self._stdin_for_cli} if self.verbose: # show live output, and immediately raise exception if process fails - print(f'> {subprocess.list2cmdline(self._cli_cmd)}') + print(f'> {subprocess.list2cmdline(self._cli_cmd)}', flush=True) run_kwargs['check'] = True else: # capture output, and only print if there's an error @@ -288,7 +288,8 @@ def run(self): f'Gb/s:{bytes_to_gigabit(bytes_per_run) / run_secs:.3f} ' + f'Mb/s:{bytes_to_megabit(bytes_per_run) / run_secs:.3f} ' + f'GiB/s:{bytes_to_GiB(bytes_per_run) / run_secs:.3f} ' + - f'MiB/s:{bytes_to_MiB(bytes_per_run) / run_secs:.3f}') + f'MiB/s:{bytes_to_MiB(bytes_per_run) / run_secs:.3f}', + flush=True) # Break out if we've exceeded max_repeat_secs app_secs = ns_to_secs(time.perf_counter_ns() - app_start_ns) diff --git a/runners/s3-benchrunner-cli/scripts/build.py b/runners/s3-benchrunner-cli/scripts/build.py index 1a7b2446..d6e621ec 100755 --- a/runners/s3-benchrunner-cli/scripts/build.py +++ b/runners/s3-benchrunner-cli/scripts/build.py @@ -17,7 +17,7 @@ def run(cmd_args: list[str]): - print(f'> {subprocess.list2cmdline(cmd_args)}') + print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True) subprocess.run(cmd_args, check=True) diff --git a/runners/s3-benchrunner-crt-java/scripts/build.py b/runners/s3-benchrunner-crt-java/scripts/build.py index a39bbb9a..fa7c2798 100755 --- a/runners/s3-benchrunner-crt-java/scripts/build.py +++ b/runners/s3-benchrunner-crt-java/scripts/build.py @@ -17,7 +17,7 @@ def run(cmd_args: list[str]): - print(f'> {subprocess.list2cmdline(cmd_args)}') + print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True) subprocess.run(cmd_args, check=True) diff --git a/runners/s3-benchrunner-crt-java/scripts/lint.py b/runners/s3-benchrunner-crt-java/scripts/lint.py index ef6d14f7..16ffebf3 100755 --- a/runners/s3-benchrunner-crt-java/scripts/lint.py +++ b/runners/s3-benchrunner-crt-java/scripts/lint.py @@ -8,7 +8,7 @@ def run(cmd_args: list[str]): - print(f'> {subprocess.list2cmdline(cmd_args)}') + print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True) if subprocess.run(cmd_args).returncode != 0: exit('FAILED') diff --git a/runners/s3-benchrunner-python/benchrunner.py b/runners/s3-benchrunner-python/benchrunner.py index 998199fe..e3193d9b 100755 --- a/runners/s3-benchrunner-python/benchrunner.py +++ b/runners/s3-benchrunner-python/benchrunner.py @@ -143,12 +143,12 @@ def __init__(self, *args, **kwargs): def run(self): # kick off all tasks - futures = [self._make_request(i) - for i in range(len(self.config.tasks))] + requests = [self._make_request(i) + for i in range(len(self.config.tasks))] # wait until all tasks are done - for future in futures: - future.result() + for request in requests: + request.finished_future.result() def _make_request(self, task_i) -> Future: task = self.config.tasks[task_i] @@ -179,8 +179,6 @@ def _make_request(self, task_i) -> Future: if self.config.files_on_disk: recv_filepath = task.key - future: Future[None] = Future() - # completion callback sets the future as complete, # or exits the program on error def on_done(error: Optional[Exception], @@ -200,11 +198,7 @@ def on_done(error: Optional[Exception], if error_body is not None: print(error_body) - future.set_exception(error) - else: - future.set_result(None) - - self._s3_client.make_request( + return self._s3_client.make_request( request=awscrt.http.HttpRequest( method, path, headers, send_stream), type=s3type, @@ -212,8 +206,6 @@ def on_done(error: Optional[Exception], send_filepath=send_filepath, on_done=on_done) - return future - class Boto3Benchmark(Benchmark): def __init__(self, *args, **kwargs): @@ -289,7 +281,8 @@ def write(self, b): f'Gb/s:{bytes_to_gigabit(bytes_per_run) / run_secs:.3f} ' + f'Mb/s:{bytes_to_megabit(bytes_per_run) / run_secs:.3f} ' + f'GiB/s:{bytes_to_GiB(bytes_per_run) / run_secs:.3f} ' + - f'MiB/s:{bytes_to_MiB(bytes_per_run) / run_secs:.3f}') + f'MiB/s:{bytes_to_MiB(bytes_per_run) / run_secs:.3f}', + flush=True) # Break out if we've exceeded max_repeat_secs app_secs = ns_to_secs(time.perf_counter_ns() - app_start_ns) diff --git a/scripts/fetch-git-repo.py b/scripts/fetch-git-repo.py index 7e912f8f..dfaddc33 100755 --- a/scripts/fetch-git-repo.py +++ b/scripts/fetch-git-repo.py @@ -21,12 +21,12 @@ def run(cmd_args: list[str]): - print(f'> {subprocess.list2cmdline(cmd_args)}') + print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True) subprocess.run(cmd_args, check=True) def try_run(cmd_args: list[str]) -> bool: - print(f'> {subprocess.list2cmdline(cmd_args)}') + print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True) result = subprocess.run(cmd_args) return result.returncode == 0 diff --git a/scripts/install-tools-AL2023.py b/scripts/install-tools-AL2023.py index 9d96ab32..8a2a0087 100755 --- a/scripts/install-tools-AL2023.py +++ b/scripts/install-tools-AL2023.py @@ -8,7 +8,7 @@ def run(cmd_args: list[str]): - print(f'> {subprocess.list2cmdline(cmd_args)}') + print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True) subprocess.run(cmd_args, check=True) diff --git a/scripts/lint-python.py b/scripts/lint-python.py index 06b5e2b8..bb236d08 100755 --- a/scripts/lint-python.py +++ b/scripts/lint-python.py @@ -9,7 +9,7 @@ def run(cmd_args: list[str]): - print(f'> {subprocess.list2cmdline(cmd_args)}') + print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True) result = subprocess.run(cmd_args) if result.returncode != 0: exit('FAILED') diff --git a/scripts/run-benchmarks.py b/scripts/run-benchmarks.py index 46a01725..b1e1f129 100755 --- a/scripts/run-benchmarks.py +++ b/scripts/run-benchmarks.py @@ -3,6 +3,7 @@ from pathlib import Path import shlex import subprocess +import sys parser = argparse.ArgumentParser( description='Run benchmarks with a specific runner') @@ -22,7 +23,13 @@ '--benchmark', action='append', help='Path to specific benchmark JSON file. ' + 'May be specified multiple times. ' + - 'By default, everything in benchmarks/ is run.') + 'If omitted, everything in benchmarks/ is run.') +parser.add_argument( + '--files-dir', + help='Launch runner in this directory. ' + + 'Files are uploaded from and downloaded to here' + + 'If omitted, CWD is used.') + args = parser.parse_args() if args.benchmark: @@ -40,18 +47,23 @@ if not benchmark.exists(): exit(f'benchmark not found: {str(benchmark)}') + files_dir = args.files_dir if args.files_dir else str(Path.cwd()) + # split using shell-like syntax, # in case runner-cmd has weird stuff like quotes, spaces, etc cmd = shlex.split(args.runner_cmd) cmd += [str(benchmark), args.bucket, args.region, str(args.throughput)] - print(f'> {subprocess.list2cmdline(cmd)}') - run = subprocess.run(cmd, text=True) + print(f'> {subprocess.list2cmdline(cmd)}', flush=True) + run = subprocess.run(cmd, text=True, cwd=files_dir) # if runner skipped the benchmark, keep going if run.returncode == 123: continue - # TODO: keep going or not? + # if runner failed and we're only running 1 benchmark, exit with failure + # but if we're running multiple benchmarks, keep going if run.returncode != 0: - exit('benchmark failed') + print('benchmark failed') + if len(benchmarks) == 1: + exit(1)