flush stdout before running subprocess, so things print in order

runners/s3-benchrunner-c/scripts/build.py

@@ -31,7 +31,7 @@
 
 
 def run(cmd_args: list[str]):
-    print(f'> {subprocess.list2cmdline(cmd_args)}')
+    print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True)
     subprocess.run(cmd_args, check=True)
 
 

runners/s3-benchrunner-cli/benchrunner.py

@@ -234,7 +234,7 @@ def _using_all_files_in_dir(self, action: str, prefix: str) -> bool:
             paginator = s3.get_paginator('list_objects_v2')
             for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
                 for obj in page['Contents']:
-                    if obj['Key'] in all_task_keys:
+                    if not obj['Key'] in all_task_keys:
                         return False
 
         else:  # upload
@@ -252,7 +252,7 @@ def run(self):
                       'input': self._stdin_for_cli}
         if self.verbose:
             # show live output, and immediately raise exception if process fails
-            print(f'> {subprocess.list2cmdline(self._cli_cmd)}')
+            print(f'> {subprocess.list2cmdline(self._cli_cmd)}', flush=True)
             run_kwargs['check'] = True
         else:
             # capture output, and only print if there's an error
@@ -288,7 +288,8 @@ def run(self):
               f'Gb/s:{bytes_to_gigabit(bytes_per_run) / run_secs:.3f} ' +
               f'Mb/s:{bytes_to_megabit(bytes_per_run) / run_secs:.3f} ' +
               f'GiB/s:{bytes_to_GiB(bytes_per_run) / run_secs:.3f} ' +
-              f'MiB/s:{bytes_to_MiB(bytes_per_run) / run_secs:.3f}')
+              f'MiB/s:{bytes_to_MiB(bytes_per_run) / run_secs:.3f}',
+              flush=True)
 
         # Break out if we've exceeded max_repeat_secs
         app_secs = ns_to_secs(time.perf_counter_ns() - app_start_ns)

runners/s3-benchrunner-cli/scripts/build.py

@@ -17,7 +17,7 @@
 
 
 def run(cmd_args: list[str]):
-    print(f'> {subprocess.list2cmdline(cmd_args)}')
+    print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True)
     subprocess.run(cmd_args, check=True)
 
 

runners/s3-benchrunner-crt-java/scripts/build.py

@@ -17,7 +17,7 @@
 
 
 def run(cmd_args: list[str]):
-    print(f'> {subprocess.list2cmdline(cmd_args)}')
+    print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True)
     subprocess.run(cmd_args, check=True)
 
 

runners/s3-benchrunner-crt-java/scripts/lint.py

@@ -8,7 +8,7 @@
 
 
 def run(cmd_args: list[str]):
-    print(f'> {subprocess.list2cmdline(cmd_args)}')
+    print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True)
     if subprocess.run(cmd_args).returncode != 0:
         exit('FAILED')
 

runners/s3-benchrunner-python/benchrunner.py

@@ -143,12 +143,12 @@ def __init__(self, *args, **kwargs):
 
     def run(self):
         # kick off all tasks
-        futures = [self._make_request(i)
-                   for i in range(len(self.config.tasks))]
+        requests = [self._make_request(i)
+                    for i in range(len(self.config.tasks))]
 
         # wait until all tasks are done
-        for future in futures:
-            future.result()
+        for request in requests:
+            request.finished_future.result()
 
     def _make_request(self, task_i) -> Future:
         task = self.config.tasks[task_i]
@@ -179,8 +179,6 @@ def _make_request(self, task_i) -> Future:
             if self.config.files_on_disk:
                 recv_filepath = task.key
 
-        future: Future[None] = Future()
-
         # completion callback sets the future as complete,
         # or exits the program on error
         def on_done(error: Optional[Exception],
@@ -200,20 +198,14 @@ def on_done(error: Optional[Exception],
                 if error_body is not None:
                     print(error_body)
 
-                future.set_exception(error)
-            else:
-                future.set_result(None)
-
-        self._s3_client.make_request(
+        return self._s3_client.make_request(
             request=awscrt.http.HttpRequest(
                 method, path, headers, send_stream),
             type=s3type,
             recv_filepath=recv_filepath,
             send_filepath=send_filepath,
             on_done=on_done)
 
-        return future
-
 
 class Boto3Benchmark(Benchmark):
     def __init__(self, *args, **kwargs):
@@ -289,7 +281,8 @@ def write(self, b):
               f'Gb/s:{bytes_to_gigabit(bytes_per_run) / run_secs:.3f} ' +
               f'Mb/s:{bytes_to_megabit(bytes_per_run) / run_secs:.3f} ' +
               f'GiB/s:{bytes_to_GiB(bytes_per_run) / run_secs:.3f} ' +
-              f'MiB/s:{bytes_to_MiB(bytes_per_run) / run_secs:.3f}')
+              f'MiB/s:{bytes_to_MiB(bytes_per_run) / run_secs:.3f}',
+              flush=True)
 
         # Break out if we've exceeded max_repeat_secs
         app_secs = ns_to_secs(time.perf_counter_ns() - app_start_ns)

scripts/fetch-git-repo.py

@@ -21,12 +21,12 @@
 
 
 def run(cmd_args: list[str]):
-    print(f'> {subprocess.list2cmdline(cmd_args)}')
+    print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True)
     subprocess.run(cmd_args, check=True)
 
 
 def try_run(cmd_args: list[str]) -> bool:
-    print(f'> {subprocess.list2cmdline(cmd_args)}')
+    print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True)
     result = subprocess.run(cmd_args)
     return result.returncode == 0
 

scripts/install-tools-AL2023.py

@@ -8,7 +8,7 @@
 
 
 def run(cmd_args: list[str]):
-    print(f'> {subprocess.list2cmdline(cmd_args)}')
+    print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True)
     subprocess.run(cmd_args, check=True)
 
 

scripts/lint-python.py

@@ -9,7 +9,7 @@
 
 
 def run(cmd_args: list[str]):
-    print(f'> {subprocess.list2cmdline(cmd_args)}')
+    print(f'> {subprocess.list2cmdline(cmd_args)}', flush=True)
     result = subprocess.run(cmd_args)
     if result.returncode != 0:
         exit('FAILED')

scripts/run-benchmarks.py

@@ -3,6 +3,7 @@
 from pathlib import Path
 import shlex
 import subprocess
+import sys
 
 parser = argparse.ArgumentParser(
     description='Run benchmarks with a specific runner')
@@ -22,7 +23,13 @@
     '--benchmark', action='append',
     help='Path to specific benchmark JSON file. ' +
     'May be specified multiple times. ' +
-    'By default, everything in benchmarks/ is run.')
+    'If omitted, everything in benchmarks/ is run.')
+parser.add_argument(
+    '--files-dir',
+    help='Launch runner in this directory. ' +
+    'Files are uploaded from and downloaded to here' +
+    'If omitted, CWD is used.')
+
 args = parser.parse_args()
 
 if args.benchmark:
@@ -40,18 +47,23 @@
     if not benchmark.exists():
         exit(f'benchmark not found: {str(benchmark)}')
 
+    files_dir = args.files_dir if args.files_dir else str(Path.cwd())
+
     # split using shell-like syntax,
     # in case runner-cmd has weird stuff like quotes, spaces, etc
     cmd = shlex.split(args.runner_cmd)
 
     cmd += [str(benchmark), args.bucket, args.region, str(args.throughput)]
-    print(f'> {subprocess.list2cmdline(cmd)}')
-    run = subprocess.run(cmd, text=True)
+    print(f'> {subprocess.list2cmdline(cmd)}', flush=True)
+    run = subprocess.run(cmd, text=True, cwd=files_dir)
 
     # if runner skipped the benchmark, keep going
     if run.returncode == 123:
         continue
 
-    # TODO: keep going or not?
+    # if runner failed and we're only running 1 benchmark, exit with failure
+    # but if we're running multiple benchmarks, keep going
     if run.returncode != 0:
-        exit('benchmark failed')
+        print('benchmark failed')
+        if len(benchmarks) == 1:
+            exit(1)