feat: Export directory prefix

hpc-io · Jun 7, 2024 · a7090eb · a7090eb
1 parent 3f83dcc
commit a7090eb
Show file tree

Hide file tree

Showing 4 changed files with 113 additions and 80 deletions.
diff --git a/drishti/handlers/handle_darshan.py b/drishti/handlers/handle_darshan.py
@@ -1,17 +1,18 @@
 #!/usr/bin/env python3
 
 import io
-import sys
-import time
 import shlex
 import shutil
 import subprocess
-import pandas as pd
+import sys
+import time
+
 import darshan
 import darshan.backend.cffi_backend as darshanll
-
-from rich import print
+import pandas as pd
 from packaging import version
+from rich import print
+
 from drishti.includes.module import *
 
 
@@ -752,15 +753,12 @@ def handler():
     display_thresholds(console)
     display_footer(console, insights_start_time, insights_end_time)
 
-    filename = '{}.html'.format(args.log_path)
-    export_html(console, filename)
+    input_filename = os.path.basename(args.log_path).replace('.darshan', '')
+    out_dir = args.export_dir if args.export_dir != "" else os.getcwd()
 
-    filename = '{}.svg'.format(args.log_path)
-    export_svg(console, filename)
+    print(f"DEBUG: outfile_name: {input_filename}")
 
-    filename = '{}-summary.csv'.format(
-        args.log_path.replace('.darshan', '')
-    )
-
-    export_csv(filename, job['job']['jobid'])
+    export_html(console, out_dir, input_filename)
+    export_svg(console, out_dir, input_filename)
 
+    export_csv(out_dir, input_filename, job['job']['jobid'])
diff --git a/drishti/includes/config.py b/drishti/includes/config.py
@@ -1,13 +1,12 @@
 #!/usr/bin/env python3
 
-import os
 import json
+import os
 
 from rich.console import Console, Group
 from rich.padding import Padding
 from rich.panel import Panel
-from rich.terminal_theme import TerminalTheme
-from rich.terminal_theme import MONOKAI
+from rich.terminal_theme import MONOKAI, TerminalTheme
 
 from drishti.includes.parser import *
 

diff --git a/drishti/includes/module.py b/drishti/includes/module.py
@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
-import datetime
 import csv
+import datetime
 import time
+
 import pandas as pd
 from rich import box
 from rich.syntax import Syntax
+
 from drishti.includes.config import *
 
 '''
@@ -1823,76 +1825,99 @@ def display_footer(console, insights_start_time, insights_end_time):
         )
     )
 
-def export_html(console, filename):
+def export_html(console, export_dir, filename):
     '''
     '''
 
-    if args.export_html:
-        console.save_html(
-            filename,
-            theme=set_export_theme(),
-            clear=False
-        )
+    if not args.export_html:
+        print("DEBUG: export_html() - return")
+        return
 
+    os.makedirs(export_dir, exist_ok=True)
+    filepath = os.path.join(export_dir, f"{filename}.html")
 
-def export_svg(console, filename):
-    if args.export_svg:
-        console.save_svg(
-            filename,
-            title='Drishti',
-            theme=set_export_theme(),
-            clear=False
-        )
+    console.save_html(
+        filepath,
+        theme=set_export_theme(),
+        clear=False
+    )
 
+    print("DEBUG: END export_html()")
 
-def export_csv(filename, jobid=None):
-    if args.export_csv:
-        issues = [
-            'JOB',
-            INSIGHTS_STDIO_HIGH_USAGE,
-            INSIGHTS_POSIX_WRITE_COUNT_INTENSIVE,
-            INSIGHTS_POSIX_READ_COUNT_INTENSIVE,
-            INSIGHTS_POSIX_WRITE_SIZE_INTENSIVE,
-            INSIGHTS_POSIX_READ_SIZE_INTENSIVE,
-            INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_USAGE,
-            INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_USAGE,
-            INSIGHTS_POSIX_HIGH_MISALIGNED_MEMORY_USAGE,
-            INSIGHTS_POSIX_HIGH_MISALIGNED_FILE_USAGE,
-            INSIGHTS_POSIX_REDUNDANT_READ_USAGE,
-            INSIGHTS_POSIX_REDUNDANT_WRITE_USAGE,
-            INSIGHTS_POSIX_HIGH_RANDOM_READ_USAGE,
-            INSIGHTS_POSIX_HIGH_SEQUENTIAL_READ_USAGE,
-            INSIGHTS_POSIX_HIGH_RANDOM_WRITE_USAGE,
-            INSIGHTS_POSIX_HIGH_SEQUENTIAL_WRITE_USAGE,
-            INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_SHARED_FILE_USAGE,
-            INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_SHARED_FILE_USAGE,
-            INSIGHTS_POSIX_HIGH_METADATA_TIME,
-            INSIGHTS_POSIX_SIZE_IMBALANCE,
-            INSIGHTS_POSIX_TIME_IMBALANCE,
-            INSIGHTS_POSIX_INDIVIDUAL_WRITE_SIZE_IMBALANCE,
-            INSIGHTS_POSIX_INDIVIDUAL_READ_SIZE_IMBALANCE,
-            INSIGHTS_MPI_IO_NO_USAGE,
-            INSIGHTS_MPI_IO_NO_COLLECTIVE_READ_USAGE,
-            INSIGHTS_MPI_IO_NO_COLLECTIVE_WRITE_USAGE,
-            INSIGHTS_MPI_IO_COLLECTIVE_READ_USAGE,
-            INSIGHTS_MPI_IO_COLLECTIVE_WRITE_USAGE,
-            INSIGHTS_MPI_IO_BLOCKING_READ_USAGE,
-            INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE,
-            INSIGHTS_MPI_IO_AGGREGATORS_INTRA,
-            INSIGHTS_MPI_IO_AGGREGATORS_INTER,
-            INSIGHTS_MPI_IO_AGGREGATORS_OK
-        ]
-        if codes:
-            issues.extend(codes)
 
-        detected_issues = dict.fromkeys(issues, False)
-        detected_issues['JOB'] = jobid
+def export_svg(console, export_dir, filename):
+    if not args.export_svg:
+        return
+
+    os.makedirs(export_dir, exist_ok=True)
+    filepath = os.path.join(export_dir, f"{filename}.svg")
+
+    console.save_svg(
+        filepath,
+        title='Drishti',
+        theme=set_export_theme(),
+        clear=False
+    )
 
-        for report in csv_report:
-            detected_issues[report] = True
 
-        with open(filename, 'w') as f:
-            w = csv.writer(f)
-            w.writerow(detected_issues.keys())
-            w.writerow(detected_issues.values())
+def export_csv(export_dir, filename, jobid=None):
+    if not args.export_csv:
+        return
+
+    issues = [
+        'JOB',
+        INSIGHTS_STDIO_HIGH_USAGE,
+        INSIGHTS_POSIX_WRITE_COUNT_INTENSIVE,
+        INSIGHTS_POSIX_READ_COUNT_INTENSIVE,
+        INSIGHTS_POSIX_WRITE_SIZE_INTENSIVE,
+        INSIGHTS_POSIX_READ_SIZE_INTENSIVE,
+        INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_USAGE,
+        INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_USAGE,
+        INSIGHTS_POSIX_HIGH_MISALIGNED_MEMORY_USAGE,
+        INSIGHTS_POSIX_HIGH_MISALIGNED_FILE_USAGE,
+        INSIGHTS_POSIX_REDUNDANT_READ_USAGE,
+        INSIGHTS_POSIX_REDUNDANT_WRITE_USAGE,
+        INSIGHTS_POSIX_HIGH_RANDOM_READ_USAGE,
+        INSIGHTS_POSIX_HIGH_SEQUENTIAL_READ_USAGE,
+        INSIGHTS_POSIX_HIGH_RANDOM_WRITE_USAGE,
+        INSIGHTS_POSIX_HIGH_SEQUENTIAL_WRITE_USAGE,
+        INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_SHARED_FILE_USAGE,
+        INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_SHARED_FILE_USAGE,
+        INSIGHTS_POSIX_HIGH_METADATA_TIME,
+        INSIGHTS_POSIX_SIZE_IMBALANCE,
+        INSIGHTS_POSIX_TIME_IMBALANCE,
+        INSIGHTS_POSIX_INDIVIDUAL_WRITE_SIZE_IMBALANCE,
+        INSIGHTS_POSIX_INDIVIDUAL_READ_SIZE_IMBALANCE,
+        INSIGHTS_MPI_IO_NO_USAGE,
+        INSIGHTS_MPI_IO_NO_COLLECTIVE_READ_USAGE,
+        INSIGHTS_MPI_IO_NO_COLLECTIVE_WRITE_USAGE,
+        INSIGHTS_MPI_IO_COLLECTIVE_READ_USAGE,
+        INSIGHTS_MPI_IO_COLLECTIVE_WRITE_USAGE,
+        INSIGHTS_MPI_IO_BLOCKING_READ_USAGE,
+        INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE,
+        INSIGHTS_MPI_IO_AGGREGATORS_INTRA,
+        INSIGHTS_MPI_IO_AGGREGATORS_INTER,
+        INSIGHTS_MPI_IO_AGGREGATORS_OK
+    ]
+    if codes:
+        issues.extend(codes)
+
+    detected_issues = dict.fromkeys(issues, False)
+    detected_issues['JOB'] = jobid
+
+    for report in csv_report:
+        detected_issues[report] = True
+
+    # ensure dir exists
+    os.makedirs(export_dir, exist_ok=True)
+    filepath = os.path.join(export_dir, f"{filename}.csv")
+
+    print(f"DEBUG: export_dir: {export_dir}")
+    print(f"DEBUG: filename: {filename}")
+    print(f"DEBUG: filepath: {filepath}")
+
+    with open(filepath, 'w') as f:
+        w = csv.writer(f)
+        w.writerow(detected_issues.keys())
+        w.writerow(detected_issues.values())
 
diff --git a/drishti/includes/parser.py b/drishti/includes/parser.py
@@ -96,6 +96,13 @@
     help='Export a CSV with the code of all issues that were triggered'
 )
 
+parser.add_argument(
+    '--export_dir',
+    default="",
+    dest='export_dir',
+    help='Specify the directory prefix for the output files (if any)'
+)
+
 parser.add_argument(
     '--json', 
     default=False, 
@@ -119,3 +126,7 @@
 )
 
 args = parser.parse_args()
+
+print(f"DEBUG: log_path: {args.log_path}")
+print(f"DEBUG: export_path: {args.export_dir}")
+print(f"DEBUG: export_csv: {args.export_csv}")