Handling the measurements associated with multiple runs

include/onnx-mlir/Compiler/OMCompilerRuntimeTypes.h

@@ -30,7 +30,8 @@ typedef enum {
   InstrumentBeforeOp = 0x0,
   InstrumentAfterOp = 0x1,
   InstrumentReportTime = 0x2,
-  InstrumentReportMemory = 0x4,
+  InstrumentReportMemory = 0x3,
+  InstrumentInit = 0x4,
 } InstrumentActions;
 
 /* Definition of setter/getter from a 64 bit unsigned int. Use 64 bit only to
@@ -67,6 +68,11 @@ typedef enum {
 #define IS_INSTRUMENT_REPORT_MEMORY(x)                                         \
   ((x) & (0x1ull << (unsigned int)InstrumentReportMemory))
 
+#define SET_INSTRUMENT_INIT(x)                                        \
+  (x) = (x) | (0x1ull << (unsigned int)InstrumentInit)
+#define IS_INSTRUMENT_INIT(x)                                         \
+  ((x) & (0x1ull << (unsigned int)InstrumentInit))
+
 /* Second - third byte. */
 #define INSTRUMENT_OP_NAME_MASK 0x3Full /* Max 64 chars. */
 #define SET_INSTRUMENT_OP_NAME_LEN(x, len)                                     \

src/Accelerators/Accelerator.hpp

@@ -108,8 +108,8 @@ class Accelerator {
   //===--------------------------------------------------------------------===//
 
   /// Convert TensorType to MemRefType.
-  /// Acccelators may have special versions of TensorType. If not, override this
-  /// method and return nullptr.
+  /// Accelerators may have special versions of TensorType. If not, override
+  /// this method and return nullptr.
   virtual mlir::MemRefType convertTensorTypeToMemRefType(
       const mlir::TensorType tensorType) const = 0;
 

src/Runtime/OMInstrument.inc

@@ -165,7 +165,7 @@ FILE *getInstrumentFile() {
   fout = stdout;
   if (getenv("ONNX_MLIR_INSTRUMENT_FILE")) {
     char *fileName = getenv("ONNX_MLIR_INSTRUMENT_FILE");
-    FILE *newFileHandle = fopen(fileName, "w");
+    FILE *newFileHandle = fopen(fileName, "w+");
     if (newFileHandle) {
       fout = newFileHandle;
     }
@@ -193,28 +193,26 @@ void OMInstrumentInit() {
   if (!instrumentReportDisabled) {
     TimeInit();
   }
+
+  fprintf(fout, "==START-REPORT==\n");
 }
 
 void OMInstrumentPoint(const char *opName, int64_t iTag, const char *nodeName) {
   if (instrumentReportDisabled)
     return;
 
-  // Initialize on first call. Would prefer to call explicitly, but there is
-  // currently no support for that.
-  static bool firstTime = true;
-  if (firstTime) {
-    OMInstrumentInit();
-    firstTime = false;
-  }
-
   // Detect which reporting we have to do here.
   uint64_t tag = iTag;
+  bool initInstrument = IS_INSTRUMENT_INIT(tag);
   bool isBefore = IS_INSTRUMENT_BEFORE_OP(tag);
   bool reportTime =
       !instrumentReportTimeDisabled && IS_INSTRUMENT_REPORT_TIME(tag);
   bool reportMem =
       !instrumentReportMemoryDisabled && IS_INSTRUMENT_REPORT_MEMORY(tag);
 
+  if (initInstrument)
+    OMInstrumentInit();
+
   if (!reportTime && !reportMem) {
     fprintf(fout, "==TICK-REPORT==, %i\n", instrumentCounter++);
     return;

src/Transform/ONNX/InstrumentPass.cpp

@@ -132,6 +132,7 @@ class InstrumentPass
     if (instrumentOps == "" || instrumentOps == "NONE")
       return;
     init(instrumentOps);
+    bool hasInitializedRuntime = false;
 
     // Iterate on the operations nested in this function
     getOperation().walk([&](mlir::Operation *op) -> WalkResult {
@@ -154,13 +155,24 @@ class InstrumentPass
         if (std::regex_match(opName, re)) {
           Location loc = op->getLoc();
           OpBuilder opBuilder(op);
-          if (instrumentBefore)
-            opBuilder.create<mlir::KrnlInstrumentOp>(loc, op, beforeTag());
+          if (instrumentBefore) {
+            uint64_t tag = beforeTag();
+            if (!hasInitializedRuntime) {
+              SET_INSTRUMENT_INIT(tag);
+              hasInitializedRuntime = true;
+            }
+            opBuilder.create<mlir::KrnlInstrumentOp>(loc, op, tag);
+          }
 
           // Can not insert after Op (e.g. ONNXYieldOP) with IsTerminator Trait
           if (instrumentAfter && !op->hasTrait<OpTrait::IsTerminator>()) {
             opBuilder.setInsertionPointAfter(op);
-            opBuilder.create<mlir::KrnlInstrumentOp>(loc, op, afterTag());
+            uint64_t tag = afterTag();
+            if (!hasInitializedRuntime) {
+              SET_INSTRUMENT_INIT(tag);
+              hasInitializedRuntime = true;
+            }
+            opBuilder.create<mlir::KrnlInstrumentOp>(loc, op, tag);
           }
         }
       }

test/mlir/accelerators/nnpa/conversion/instrument/add-onnx-level.mlir

@@ -1,12 +1,14 @@
-// RUN: onnx-mlir --mcpu=z16 --maccel=NNPA  --printIR --EmitZHighIR --instrument-stage=Onnx --instrument-ops=onnx.* --InstrumentBeforeOp --InstrumentAfterOp --InstrumentReportTime -tag="test" %s | FileCheck %s
+// RUN: onnx-mlir --mcpu=z16 --maccel=NNPA  --printIR --EmitZHighIR -profile-ir=Onnx %s | FileCheck %s
+
+// -----
 
 func.func @test_instrument_add_onnx(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10x10xf32>) -> tensor<*xf32> {
   %0 = "onnx.Add"(%arg0, %arg1) {onnx_node_name = "onnx.Add"} : (tensor<10x10xf32>, tensor<10x10xf32>) -> tensor<*xf32>
   "onnx.Return"(%0) : (tensor<*xf32>) -> ()
 }
 
 // CHECK-LABEL:  func.func @test_instrument_add_onnx
-// CHECK:           "krnl.runtime_instrument"() {nodeName = "onnx.Add", opName = "onnx.Add", tag = 5 : i64} : () -> ()
+// CHECK:           "krnl.runtime_instrument"() {nodeName = "onnx.Add", opName = "onnx.Add", tag = 21 : i64} : () -> ()
 // CHECK:           "zhigh.Stick"
 // CHECK:           "zhigh.Stick"
 // CHECK:           "zhigh.Add"

test/mlir/accelerators/nnpa/conversion/instrument/add-onnx-zhigh-level.mlir

@@ -1,4 +1,6 @@
-// RUN: onnx-mlir --mcpu=z16 --maccel=NNPA --printIR --EmitZHighIR --instrument-stage=ZHigh --instrument-ops="onnx.*,zhigh.*" --InstrumentBeforeOp --InstrumentAfterOp --InstrumentReportTime -tag="test" %s  | FileCheck %s
+// RUN: onnx-mlir --mcpu=z16 --maccel=NNPA --printIR --EmitZHighIR --profile-ir=ZHigh %s  | FileCheck %s
+
+// -----
 
 func.func @test_instrument_add_onnx_zhigh(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10x1xf32>) -> tensor<*xf32> {
   %0 = "onnx.Add"(%arg0, %arg1) {onnx_node_name = "onnx.Add1"} : (tensor<10x10xf32>, tensor<10x1xf32>) -> tensor<*xf32>
@@ -7,7 +9,7 @@ func.func @test_instrument_add_onnx_zhigh(%arg0 : tensor<10x10xf32>, %arg1 : ten
 }
 
 // CHECK-LABEL:  func.func @test_instrument_add_onnx_zhigh
-// CHECK:           "krnl.runtime_instrument"() {nodeName = "onnx.Add1", opName = "onnx.Add", tag = 5 : i64} : () -> ()
+// CHECK:           "krnl.runtime_instrument"() {nodeName = "onnx.Add1", opName = "onnx.Add", tag = 21 : i64} : () -> ()
 // CHECK:           "onnx.Add"
 // CHECK:           "krnl.runtime_instrument"() {nodeName = "onnx.Add1", opName = "onnx.Add", tag = 6 : i64} : () -> ()
 // CHECK:           "krnl.runtime_instrument"() {opName = "zhigh.Stick", tag = 5 : i64} : () -> ()

test/mlir/accelerators/nnpa/conversion/instrument/add-zhigh-level.mlir

@@ -1,12 +1,14 @@
-// RUN: onnx-mlir --mcpu=z16 --maccel=NNPA --printIR --EmitZLowIR --instrument-stage=ZHigh --instrument-ops="zhigh.*" --InstrumentBeforeOp --InstrumentAfterOp --InstrumentReportTime -tag="test" %s  | FileCheck %s
+// RUN: onnx-mlir --mcpu=z16 --maccel=NNPA --printIR --EmitZLowIR --instrument-stage=ZHigh --instrument-ops=zhigh.* --InstrumentBeforeOp --InstrumentAfterOp --InstrumentReportTime %s  | FileCheck %s
+
+// -----
 
 func.func @test_instrument_add_zhigh(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10x10xf32>) -> tensor<*xf32> {
   %0 = "onnx.Add"(%arg0, %arg1) : (tensor<10x10xf32>, tensor<10x10xf32>) -> tensor<*xf32>
   "onnx.Return"(%0) : (tensor<*xf32>) -> ()
 }
 
 // CHECK-LABEL:  func.func @test_instrument_add_zhigh
-// CHECK:           "krnl.runtime_instrument"() {opName = "zhigh.Stick", tag = 5 : i64} : () -> ()
+// CHECK:           "krnl.runtime_instrument"() {opName = "zhigh.Stick", tag = 21 : i64} : () -> ()
 // CHECK:           memref.alloc()
 // CHECK:           "zlow.stick"
 // CHECK:           "krnl.runtime_instrument"() {opName = "zhigh.Stick", tag = 6 : i64} : () -> ()

test/mlir/accelerators/nnpa/conversion/instrument/add-zlow-level.mlir

@@ -1,12 +1,14 @@
-// RUN: onnx-mlir --mcpu=z16 --maccel=NNPA --printIR --EmitZLowIR --instrument-stage=ZLow --instrument-ops="zlow.*" --InstrumentBeforeOp --InstrumentAfterOp --InstrumentReportTime -tag="test" %s  | FileCheck %s
+// RUN:  onnx-mlir --mcpu=z16 --maccel=NNPA --printIR --EmitZLowIR --instrument-stage=ZLow --instrument-ops=zlow.* --InstrumentBeforeOp --InstrumentAfterOp --InstrumentReportTime %s  | FileCheck %s
+
+// -----
 
 func.func @test_instrument_add_zlow(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10x10xf32>) -> tensor<*xf32> {
   %0 = "onnx.Add"(%arg0, %arg1) : (tensor<10x10xf32>, tensor<10x10xf32>) -> tensor<*xf32>
   "onnx.Return"(%0) : (tensor<*xf32>) -> ()
 }
 
 // CHECK-LABEL:  func.func @test_instrument_add_zlow
-// CHECK:           "krnl.runtime_instrument"() {opName = "zlow.stick", tag = 5 : i64} : () -> ()
+// CHECK:           "krnl.runtime_instrument"() {opName = "zlow.stick", tag = 21 : i64} : () -> ()
 // CHECK:           "zlow.stick"
 // CHECK:           "krnl.runtime_instrument"() {opName = "zlow.stick", tag = 6 : i64} : () -> ()
 // CHECK:           memref.alloc()
@@ -26,5 +28,3 @@ func.func @test_instrument_add_zlow(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10
 // CHECK:           "krnl.runtime_instrument"() {opName = "zlow.unstick", tag = 6 : i64} : () -> ()
 // CHECK:           return
 // CHECK:         }
-
-// -----

test/mlir/conversion/instrument/add.mlir

@@ -5,8 +5,10 @@ func.func @test_instrument_add_onnx(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10
   "onnx.Return"(%0) : (tensor<*xf32>) -> ()
 }
 
+// -----
+
 // CHECK-LABEL:  func.func @test_instrument_add_onnx
-// CHECK:           "krnl.runtime_instrument"() {nodeName = "model/add1", opName = "onnx.Add", tag = 5 : i64} : () -> ()
+// CHECK:           "krnl.runtime_instrument"() {nodeName = "model/add1", opName = "onnx.Add", tag = 21 : i64} : () -> ()
 // CHECK:           [[RES_:%.+]] = memref.alloc()
 // CHECK:           affine.for [[I_0_:%.+]] = 0 to 10 {
 // CHECK:             affine.for [[I_1_:%.+]] = 0 to 10 {
@@ -20,4 +22,3 @@ func.func @test_instrument_add_onnx(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10
 // CHECK:           return
 // CHECK:         }
 
-// -----

test/mlir/conversion/instrument/onnx_add.mlir

@@ -1,12 +1,14 @@
 // RUN: onnx-mlir --printIR --EmitMLIR --instrument-ops=onnx.Add --InstrumentBeforeOp --InstrumentAfterOp --InstrumentReportTime %s | FileCheck %s
 
+// -----
+
 func.func @test_instrument_add_onnx(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10x10xf32>) -> tensor<*xf32> {
   %0 = "onnx.Add"(%arg0, %arg1) {onnx_node_name = "model/add1"} : (tensor<10x10xf32>, tensor<10x10xf32>) -> tensor<*xf32>
   "onnx.Return"(%0) : (tensor<*xf32>) -> ()
 }
 
 // CHECK-LABEL:  func.func @test_instrument_add_onnx
-// CHECK:           "krnl.runtime_instrument"() {nodeName = "model/add1", opName = "onnx.Add", tag = 5 : i64} : () -> ()
+// CHECK:           "krnl.runtime_instrument"() {nodeName = "model/add1", opName = "onnx.Add", tag = 21 : i64} : () -> ()
 // CHECK:           [[RES_:%.+]] = memref.alloc()
 // CHECK:           affine.for [[I_0_:%.+]] = 0 to 10 {
 // CHECK:             affine.for [[I_1_:%.+]] = 0 to 10 {
@@ -20,4 +22,3 @@ func.func @test_instrument_add_onnx(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10
 // CHECK:           return
 // CHECK:         }
 
-// -----

test/mlir/conversion/instrument/onnx_all.mlir

@@ -1,12 +1,14 @@
 // RUN: onnx-mlir --printIR --EmitMLIR --instrument-ops=onnx.* --InstrumentBeforeOp --InstrumentAfterOp --InstrumentReportTime %s | FileCheck %s
 
+// -----
+
 func.func @test_instrument_add_onnx(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10x10xf32>) -> tensor<*xf32> {
   %0 = "onnx.Add"(%arg0, %arg1) {onnx_node_name = "model/add1"} : (tensor<10x10xf32>, tensor<10x10xf32>) -> tensor<*xf32>
   "onnx.Return"(%0) : (tensor<*xf32>) -> ()
 }
 
 // CHECK-LABEL:  func.func @test_instrument_add_onnx
-// CHECK:           "krnl.runtime_instrument"() {nodeName = "model/add1", opName = "onnx.Add", tag = 5 : i64} : () -> ()
+// CHECK:           "krnl.runtime_instrument"() {nodeName = "model/add1", opName = "onnx.Add", tag = 21 : i64} : () -> ()
 // CHECK:           [[RES_:%.+]] = memref.alloc()
 // CHECK:           affine.for [[I_0_:%.+]] = 0 to 10 {
 // CHECK:             affine.for [[I_1_:%.+]] = 0 to 10 {
@@ -19,5 +21,3 @@ func.func @test_instrument_add_onnx(%arg0 : tensor<10x10xf32>, %arg1 : tensor<10
 // CHECK:           "krnl.runtime_instrument"() {nodeName = "model/add1", opName = "onnx.Add", tag = 6 : i64} : () -> ()
 // CHECK:           return
 // CHECK:         }
-
-// -----

utils/RunONNXModel.py

@@ -550,6 +550,9 @@ def warning(msg):
 def data_without_top_bottom_quartile(data, percent):
     data = np.array(sorted(data))
     trim = int(percent * data.size / 100.0)
+    if trim == 0 or data.size - 2 * trim < 1:
+        # Want at least one element, return as is.
+        return data
     return data[trim:-trim]