treewide: Bump Snitch cluster and align build scripts

.gitlab-ci.yml

@@ -10,9 +10,7 @@ variables:
   CLANG_FORMAT: clang-format-10.0.1
   CC: gcc-9.2.0
   CXX: g++-9.2.0
-  VCS: vcs-2020.12
-  VERILATOR: verilator-4.110
-  QUESTA: questa-2022.3
+  QUESTA_SEPP: questa-2022.3
   LLVM_BINROOT: /usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin
   CLANG: /usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin/clang
   RISCV_GCC_VERSION: 8.3.0-2020.04.0
@@ -21,7 +19,7 @@ before_script:
   # yamllint disable rule:line-length
   - $PYTHON -m venv .venv
   - source .venv/bin/activate
-  - bender update
+  - $BENDER update
   - pip install -r python-requirements.txt
   # Install CVA6 compiler toolchain
   - curl -Ls -o riscv-gcc.tar.gz https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-$RISCV_GCC_VERSION-x86_64-linux-ubuntu14.tar.gz
@@ -52,10 +50,8 @@ occamy-single-cluster-vsim:
     - cd target/sim
     - make CFG_OVERRIDE=cfg/single-cluster.hjson rtl
     - make sw
-    - $QUESTA make bin/occamy_top.vsim
-    - $QUESTA ../../deps/snitch_cluster/util/sim/simulate.py
-        sw/run-single-cluster.yaml --simulator vsim
-        --sim-bin bin/occamy_top.vsim
+    - make bin/occamy_top.vsim
+    - ./run.py sw/run-single-cluster.yaml --simulator vsim
 
 #####################
 # Full Occamy tests #
@@ -67,7 +63,5 @@ occamy-full-vsim:
     - cd target/sim
     - make CFG_OVERRIDE=cfg/full.hjson rtl
     - make LENGTH=384 sw
-    - $QUESTA make bin/occamy_top.vsim
-    - $QUESTA ../../deps/snitch_cluster/util/sim/simulate.py
-        sw/run-full-occamy.yaml --simulator vsim
-        --sim-bin bin/occamy_top.vsim
+    - make bin/occamy_top.vsim
+    - ./run.py sw/run-full-occamy.yaml --simulator vsim

Bender.yml

@@ -29,7 +29,7 @@ dependencies:
   cva6:                   { path: hw/vendor/openhwgroup_cva6                                                }
   opentitan_peripherals:  { path: hw/vendor/pulp_platform_opentitan_peripherals                             }
   register_interface:     { git:  https://github.com/pulp-platform/register_interface.git, version: 0.3.8   }
-  snitch_cluster:         { git:  https://github.com/pulp-platform/snitch_cluster.git,     rev:     occamy  }
+  snitch_cluster:         { git:  https://github.com/pulp-platform/snitch_cluster.git,     rev:     extend/snRuntime }
   tech_cells_generic:     { git:  https://github.com/pulp-platform/tech_cells_generic.git, rev:     v0.2.11 }
 
 workspace:

target/sim/.gitignore

@@ -6,6 +6,7 @@
 /*.log
 /.*_targets_group
 /sw/**/build/
+/runs/
 
 # Auto-generated sources
 /src/

target/sim/Makefile

@@ -38,9 +38,6 @@ ADDRMAPGEN ?= $(ROOT)/util/addrmap/addrmapgen.py
 
 CLANG_FORMAT ?= $(shell which clang-format-10.0.1)
 
-VSIM = vsim
-VLOG = vlog
-
 #########################
 # Files and directories #
 #########################
@@ -526,7 +523,7 @@ clean-vlt: clean-work
 ############
 
 ${VSIM_BUILDDIR}/compile.vsim.tcl: $(VSIM_SOURCES) ${TB_SRCS} ${TB_CC_SOURCES} test/bootrom.bin | $(VSIM_BUILDDIR)
-	vlib $(dir $@)
+	$(VLIB) $(dir $@)
 	${BENDER} script vsim ${VSIM_BENDER} --vlog-arg="${VLOG_FLAGS} -work $(dir $@) " > $@
 	echo '${VLOG} -work $(dir $@) $(TB_CC_SOURCES) -ccflags "$(TB_CC_FLAGS)"' >> $@
 	echo 'return 0' >> $@

target/sim/run.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# Copyright 2023 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Luca Colagrande <[email protected]>
+
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).parent / '../../deps/snitch_cluster/util/sim'))
+from sim_utils import parser, get_simulations, run_simulations  # noqa: E402
+from Simulator import QuestaSimulator  # noqa: E402
+
+
+SIMULATORS = {
+    'vsim': QuestaSimulator(Path(__file__).parent.resolve() / 'bin/occamy_top.vsim')
+}
+
+
+def main():
+    args = parser('vsim', SIMULATORS.keys()).parse_args()
+    simulations = get_simulations(args.testlist, SIMULATORS[args.simulator], run_dir=args.run_dir)
+    return run_simulations(simulations,
+                           n_procs=args.n_procs,
+                           dry_run=args.dry_run,
+                           early_exit=args.early_exit)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

target/sim/sw/device/Makefile

@@ -13,15 +13,15 @@ TARGET ?= all
 APP_SUBDIRS = $(addprefix apps/,$(APPS))
 
 SUBDIRS  = runtime
+SUBDIRS += math
 SUBDIRS += $(APP_SUBDIRS)
 
 .PHONY: all $(SUBDIRS)
 
 all: $(SUBDIRS)
 
-runtime:
+$(SUBDIRS):
 	$(MAKE) -C $@ $(TARGET)
 
 # Explicit dependency of apps on runtime
-$(APP_SUBDIRS): runtime
-	$(MAKE) -C $@ $(TARGET)
+$(APP_SUBDIRS): runtime math

target/sim/sw/device/apps/common.mk

@@ -9,17 +9,25 @@
 MK_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
 include $(MK_DIR)/../toolchain.mk
 
-###################
-# Build variables #
-###################
+###############
+# Directories #
+###############
 
-# Directories
-BUILDDIR    = $(abspath build)
-APPSDIR     = $(abspath $(MK_DIR))
-RUNTIME_DIR = $(abspath $(MK_DIR)/../runtime)
+# Fixed paths in repository tree
+ROOT        = $(abspath $(MK_DIR)/../../../../../)
 SNITCH_ROOT = $(shell bender path snitch_cluster)
+APPSDIR     = $(abspath $(MK_DIR))
+RUNTIME_DIR = $(ROOT)/target/sim/sw/device/runtime
 SNRT_DIR    = $(SNITCH_ROOT)/sw/snRuntime
-SW_DIR      = $(abspath $(MK_DIR)/../../)
+SW_DIR      = $(ROOT)/target/sim/sw/
+MATH_DIR    = $(ROOT)/target/sim/sw/device/math
+
+# Paths relative to the app including this Makefile
+BUILDDIR    = $(abspath build)
+
+###################
+# Build variables #
+###################
 
 # Dependencies
 INCDIRS += $(RUNTIME_DIR)/src
@@ -29,14 +37,7 @@ INCDIRS += $(SNRT_DIR)/vendor/riscv-opcodes
 INCDIRS += $(SW_DIR)/shared/platform/generated
 INCDIRS += $(SW_DIR)/shared/platform
 INCDIRS += $(SW_DIR)/shared/runtime
-
-# Math library override
-INCDIRS += $(SNITCH_ROOT)/sw/math/arch/riscv64/bits/
-INCDIRS += $(SNITCH_ROOT)/sw/math/arch/generic
-INCDIRS += $(SNITCH_ROOT)/sw/math/src/include
-INCDIRS += $(SNITCH_ROOT)/sw/math/src/internal
-INCDIRS += $(SNITCH_ROOT)/sw/math/include/bits
-INCDIRS += $(SNITCH_ROOT)/sw/math/include
+INCDIRS += $(SNITCH_ROOT)/sw/blas
 
 # Linking sources
 BASE_LD       = $(abspath $(SNRT_DIR)/base.ld)
@@ -55,6 +56,10 @@ RISCV_LDFLAGS += -T$(BASE_LD)
 # Link snRuntime library
 RISCV_LDFLAGS += -L$(SNRT_LIB_DIR)
 RISCV_LDFLAGS += -l$(SNRT_LIB_NAME)
+# Link math library
+RISCV_LDFLAGS += -L$(MATH_DIR)/build
+RISCV_LDFLAGS += -lmath
+
 
 # Objcopy flags
 OBJCOPY_FLAGS  = -O binary

target/sim/sw/device/math/Makefile

@@ -0,0 +1,8 @@
+# Copyright 2023 ETH Zurich and University of Bologna.
+# Licensed under the Apache License, Version 2.0, see LICENSE for details.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Luca Colagrande <[email protected]>
+
+include ../toolchain.mk
+include $(SNITCH_ROOT)/sw/math/Makefile

target/sim/sw/device/runtime/Makefile

@@ -44,7 +44,7 @@ ALL_OUTPUTS = $(DEPS) $(LIB) $(DUMP)
 #########
 
 .PHONY: all
-all: math $(ALL_OUTPUTS)
+all: $(ALL_OUTPUTS)
 
 .PHONY: clean
 clean:
@@ -68,11 +68,6 @@ $(LIB): $(OBJS) | $(BUILDDIR)
 $(DUMP): $(LIB) | $(BUILDDIR)
 	$(RISCV_OBJDUMP) -D $< > $@
 
-# Build math library sources
-.PHONY: math
-math:
-	make -C $(SNITCH_ROOT)/sw/math all
-
 ifneq ($(MAKECMDGOALS),clean)
 -include $(DEPS)
 endif

target/sim/sw/device/runtime/src/occamy_start.c

@@ -10,12 +10,22 @@
 #define SNRT_CRT0_PRE_BARRIER
 #define SNRT_INVOKE_MAIN
 #define SNRT_CRT0_POST_BARRIER
-#define SNRT_CRT0_CALLBACK7
+#define SNRT_CRT0_EXIT
+#define SNRT_CRT0_ALTERNATE_EXIT
 
 static inline void snrt_crt0_callback3() {
     _snrt_cluster_hw_barrier = cluster_hw_barrier_addr(snrt_cluster_idx());
 }
 
-static inline void snrt_crt0_callback7() { return_to_cva6(SYNC_CLUSTERS); }
+static inline uint32_t* snrt_exit_code_destination() {
+    return soc_ctrl_scratch_ptr(3);
+}
+
+static inline void snrt_exit_default(int exit_code);
+
+static inline void snrt_exit(int exit_code) {
+    snrt_exit_default(exit_code);
+    if (snrt_global_core_idx() == 0) set_host_sw_interrupt();
+}
 
 #include "start.c"

target/sim/sw/device/runtime/src/snrt.h

@@ -27,6 +27,7 @@
 #include "cls.h"
 #include "cluster_interrupts.h"
 #include "dma.h"
+#include "dump.h"
 #include "global_interrupts.h"
 #include "occamy_device.h"
 #include "occamy_memory.h"

target/sim/sw/device/toolchain.mk

@@ -4,4 +4,6 @@
 #
 # Luca Colagrande <[email protected]>
 
-include $(shell bender path snitch_cluster)/target/snitch_cluster/sw/toolchain.mk
+BENDER ?= bender
+SNITCH_ROOT = $(shell $(BENDER) path snitch_cluster)
+include $(SNITCH_ROOT)/target/snitch_cluster/sw/toolchain.mk

target/sim/sw/host/apps/hello_world/verify.py

@@ -10,19 +10,16 @@
 import sys
 
 sys.path.append(str(Path(__file__).parent / '../../../../../../deps/snitch_cluster/util/sim/'))
-from simulate import run_simulation  # noqa: E402
+from sim_utils import run_simulations  # noqa: E402
+from Simulator import QuestaSimulator  # noqa: E402
 
-UART_LOG = str(Path(__file__).parent / '../../../../uart0.log')
+UART_LOG = 'uart0.log'
 EXPECTED_OUTPUT = "Hello world!\r\n"
 
 
 def parse_args():
     # Argument parsing
     parser = argparse.ArgumentParser(allow_abbrev=True)
-    parser.add_argument(
-        'simulator',
-        help='The simulator to be used',
-    )
     parser.add_argument(
         'sim_bin',
         help='The simulator binary to be used to start the simulation',
@@ -35,8 +32,10 @@ def parse_args():
 
 def main():
     args = parse_args()
-    cmd = f"{args.sim_bin} {args.snitch_bin}"
-    result, _ = run_simulation(cmd, args.simulator, {})
+    simulator = QuestaSimulator(args.sim_bin)
+    simulation = simulator.get_simulation({'elf': args.snitch_bin})
+    result = run_simulations([simulation])
+
     actual_output = ''
     with open(UART_LOG, 'rb') as file:
         actual_output = file.read().decode('ascii')

target/sim/sw/host/apps/offload/src/offload.c

@@ -18,9 +18,6 @@ int main() {
     // Start Snitches
     wakeup_snitches_cl();
 
-    // Wait for job done
-    wait_snitches_done();
-
-    // Exit routine
-    mcycle();
+    // Wait for job done and return Snitch exit code
+    return wait_snitches_done();
 }

target/sim/sw/host/runtime/host.c

@@ -258,9 +258,15 @@ void wakeup_master_snitches() {
 /**
  * @brief Waits until snitches are done executing
  */
-static inline void wait_snitches_done() {
+static inline int wait_snitches_done() {
     wait_sw_interrupt();
     clear_host_sw_interrupt();
+    int retval = *soc_ctrl_scratch_ptr(3);
+    // LSB signals completion
+    if (retval & 1)
+        return retval >> 1;
+    else
+        return -1;
 }
 
 static inline volatile uint32_t* get_shared_lock() {

target/sim/sw/run-full-occamy.yaml

@@ -4,8 +4,8 @@
 
 runs:
   - elf: host/apps/offload/build/offload-axpy.elf
-    cmd: ../../deps/snitch_cluster/sw/blas/axpy/verify.py {sim_bin} {elf}
-          --symbols-bin ./sw/device/apps/blas/axpy/build/axpy.elf
+    cmd: [../../../deps/snitch_cluster/sw/blas/axpy/verify.py, --symbols-bin,
+          ./device/apps/blas/axpy/build/axpy.elf, "${sim_bin}", "${elf}"]
   - elf: host/apps/offload/build/offload-gemm.elf
-    cmd: ../../deps/snitch_cluster/sw/blas/gemm/verify.py {sim_bin} {elf}
-          --symbols-bin ./sw/device/apps/blas/gemm/build/gemm.elf
+    cmd: [../../../deps/snitch_cluster/sw/blas/gemm/verify.py, --symbols-bin,
+          ./device/apps/blas/gemm/build/gemm.elf, "${sim_bin}", "${elf}"]

target/sim/sw/run-single-cluster.yaml

@@ -4,10 +4,10 @@
 
 runs:
   - elf: host/apps/offload/build/offload-axpy.elf
-    cmd: ../../deps/snitch_cluster/sw/blas/axpy/verify.py {sim_bin} {elf}
-          --symbols-bin ./sw/device/apps/blas/axpy/build/axpy.elf
+    cmd: [../../../deps/snitch_cluster/sw/blas/axpy/verify.py, --symbols-bin,
+          ./device/apps/blas/axpy/build/axpy.elf, "${sim_bin}", "${elf}"]
   - elf: host/apps/offload/build/offload-gemm.elf
-    cmd: ../../deps/snitch_cluster/sw/blas/gemm/verify.py {sim_bin} {elf}
-          --symbols-bin ./sw/device/apps/blas/gemm/build/gemm.elf
+    cmd: [../../../deps/snitch_cluster/sw/blas/gemm/verify.py, --symbols-bin,
+          ./device/apps/blas/gemm/build/gemm.elf, "${sim_bin}", "${elf}"]
   - elf: host/apps/hello_world/build/hello_world.elf
-    cmd: sw/host/apps/hello_world/verify.py {simulator} {sim_bin} {elf}
+    cmd: [./host/apps/hello_world/verify.py, "${sim_bin}", "${elf}"]