h100 mlp+attention+ar

microsoft · May 14, 2024 · 57720ca · 57720ca
1 parent 715aee2
commit 57720ca
Show file tree

Hide file tree

Showing 9 changed files with 354,783 additions and 1 deletion.
diff --git a/data/profiling/compute/h100/Qwen/Qwen-72B/attention.csv b/data/profiling/compute/h100/Qwen/Qwen-72B/attention.csv
diff --git a/data/profiling/compute/h100/codellama/CodeLlama-34b-Instruct-hf/attention.csv b/data/profiling/compute/h100/codellama/CodeLlama-34b-Instruct-hf/attention.csv
diff --git a/data/profiling/compute/h100/internlm/internlm-20b/attention.csv b/data/profiling/compute/h100/internlm/internlm-20b/attention.csv
diff --git a/data/profiling/compute/h100/meta-llama/Llama-2-70b-hf/attention.csv b/data/profiling/compute/h100/meta-llama/Llama-2-70b-hf/attention.csv
diff --git a/data/profiling/compute/h100/meta-llama/Llama-2-7b-hf/attention.csv b/data/profiling/compute/h100/meta-llama/Llama-2-7b-hf/attention.csv
diff --git a/data/profiling/compute/h100/microsoft/phi-2/attention.csv b/data/profiling/compute/h100/microsoft/phi-2/attention.csv
diff --git a/data/profiling/network/h100_nvlink/all_reduce.csv b/data/profiling/network/h100_nvlink/all_reduce.csv
diff --git a/simulator/profiling/collectives/main.py b/simulator/profiling/collectives/main.py
@@ -16,7 +16,7 @@ def parse_args():
         "--num_workers_per_node_combinations",
         type=int,
         nargs="+",
-        default=[1, 2, 4],
+        default=[1, 2, 4, 8],
     )
     parser.add_argument(
         "--output_dir",

diff --git a/simulator/profiling/cpu_overhead/main.py b/simulator/profiling/cpu_overhead/main.py
@@ -90,6 +90,7 @@ def profile_model(
             pbar.update(1)
 
     df = pd.DataFrame(results)
+    os.makedirs("f{output_dir}/{model_name}", exist_ok=True)
     df.to_csv(f"{output_dir}/{model_name}/cpu_overhead.csv")