separating load test from common regression

JuliaMolSim · Dec 22, 2023 · 542b81d · 542b81d
1 parent 97495f3
commit 542b81d
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 20 deletions.
diff --git a/.github/workflows/regression.yaml b/.github/workflows/regression.yaml
@@ -14,8 +14,15 @@ concurrency:
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 
 jobs:
-  Benchmark:
+  test:
+    name: Benchmarking ${{ matrix.description }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {description: run,  payload: benchmarks.jl  }
+          - {description: load, payload: load.jl }
     steps:
       # Remove older benchmark comment
       - name: pr-deleter
@@ -26,46 +33,63 @@ jobs:
           repo: <repo>
           user: github-actions[bot]
           issue: ${{github.event.number}}
-      - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@latest
+
+      - uses: actions/checkout@v4
+
+      - name: Setup Julia stable
+        uses: julia-actions/setup-julia@v1
+        with:
+          version: '1.9'
+          arch: x64
+
+      - uses: julia-actions/cache@v1
         with:
-          version: 1
-      - uses: julia-actions/julia-buildpkg@latest
+          include-matrix: false
+      - uses: julia-actions/julia-buildpkg@v1
+
       - name: Install dependencies
         run: |
           julia --project=benchmark -e '
           using Pkg
           Pkg.develop(PackageSpec(; path=pwd()))
           Pkg.instantiate()'
+
       - name: Run benchmarks against master
         # Remove baseline once merged. Regression tests will only work after this is merged
         # in master.
         run: |
-          julia --project=benchmark -e '
+          julia --project=benchmark -e "
             using BenchmarkCI
-            BenchmarkCI.judge(; baseline="HEAD", retune=true)'
+            baseline = \"HEAD\"
+            script = \"\$(pwd())/benchmark/${{ matrix.payload }}\"
+            BenchmarkCI.judge(; baseline, script, retune=true)"
         if: ${{ github.event_name == 'pull_request' }}
+
       - name: Run benchmarks against last release
         run: |
-          julia --project=benchmark -e '
+          julia --project=benchmark -e "
             import Pkg
-            baseline="v"*Pkg.TOML.parsefile("Project.toml")["version"]
+            baseline = \"v\" * Pkg.TOML.parsefile(\"Project.toml\")[\"version\"]
+            script = \"\$(pwd())/benchmark/${{ matrix.payload }}\"
             using BenchmarkCI
-            BenchmarkCI.judge(; baseline, retune=true)'
+            BenchmarkCI.judge(; baseline, script, retune=true)"
         if: ${{ github.event_name == 'schedule' ||
                 github.event.push.ref == 'refs/heads/master' }}
+
       - name: Print judgement
         run: |
           julia --project=benchmark -e '
             using BenchmarkCI
             BenchmarkCI.displayjudgement()'
+
       - name: Post results
         run: |
           julia --project=benchmark -e '
             using BenchmarkCI
             BenchmarkCI.postjudge()'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Is report successful
         run: |
           res=$(julia --project=benchmark -e '

diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
@@ -3,14 +3,4 @@ using TestItemRunner
 
 const SUITE = BenchmarkGroup()
 
-julia_cmd = unsafe_string(Base.JLOptions().julia_bin)
-SUITE["load"] = @benchmarkable run(`$julia_cmd \
-                                        --startup-file=no \
-                                        --project=$(Base.active_project()) \
-                                        -e 'using DFTK'`)
-SUITE["pecompilation"] =
-    @benchmarkable run(`$julia_cmd \
-                           --startup-file=no \
-                           --project=$(Base.active_project()) \
-                           -e 'Base.compilecache(Base.identify_package("DFTK"))'`)
 @run_package_tests filter=ti->(:regression ∈ ti.tags)
diff --git a/benchmark/load.jl b/benchmark/load.jl
@@ -0,0 +1,14 @@
+using BenchmarkTools
+
+const SUITE = BenchmarkGroup()
+
+julia_cmd = unsafe_string(Base.JLOptions().julia_bin)
+SUITE["load"] = @benchmarkable run(`$julia_cmd \
+                                        --startup-file=no \
+                                        --project=$(Base.active_project()) \
+                                        -e 'using DFTK'`)
+SUITE["pecompilation"] =
+    @benchmarkable run(`$julia_cmd \
+                           --startup-file=no \
+                           --project=$(Base.active_project()) \
+                           -e 'Base.compilecache(Base.identify_package("DFTK"))'`)