Contextual profile vogr

.gitignore

@@ -36,5 +36,4 @@ benchmarks/
 *.DS_Store
 external/*
 !external/custom-r
-.history
-profile
+.history

.gitlab-ci.yml

@@ -7,6 +7,10 @@ stages:
   - Benchmark
   - Report
 
+cache:
+  paths:
+  - apt-cache/
+
 variables:
   REBENCH_OPTIONS: "-df $CI_PROJECT_DIR/benchmarks.data -R"
   REBENCH_RUN: "/opt/rbenchmarking/Setup/run.sh /opt/rbenchmarking/rebench.conf /opt/rbenchmarking/Benchmarks /opt/rir/build/release"
@@ -62,23 +66,24 @@ cpp_check:
   needs:
     - rir_container
   script:
+    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive apt-get -o dir::cache::archives=apt-cache install -y -qq cppcheck
     - /opt/rir/tools/cppcheck
 
 # Run check-all in release mode  (additionally check pir types)
 test_release_1:
   image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA
   variables:
     GIT_STRATEGY: none
-    PIR_INLINER_INLINE_UNLIKELY: 1
-    PIR_MAX_INPUT_SIZE: 10000
-    PIR_INLINER_MAX_SIZE: 10000
-    PIR_LLVM_OPT_LEVEL: 1
+    PIR_LLVM_OPT_LEVEL: 0
+    PIR_OPT_LEVEL: 1
   stage: Run tests
   needs:
     - rir_container
   except:
     - schedules
   script:
+    - /opt/rir/container/install-test-deps.sh
     - cd /opt/rir/build/release
     - RIR_CHECK_PIR_TYPES=1 bin/tests
     - RIR_CHECK_PIR_TYPES=1 bin/gnur-make-tests check-devel || $SAVE_LOGS
@@ -93,16 +98,17 @@ test_release_2:
   image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA
   variables:
     GIT_STRATEGY: none
-    PIR_INLINER_INLINE_UNLIKELY: 1
-    PIR_MAX_INPUT_SIZE: 5000
-    PIR_INLINER_MAX_SIZE: 5000
-    PIR_LLVM_OPT_LEVEL: 1
+    PIR_LLVM_OPT_LEVEL: 0
+    PIR_OPT_LEVEL: 0
+    PIR_MAX_INPUT_SIZE: 4000
+    PIR_WARMUP: 5
   stage: Run tests
   needs:
     - rir_container
   except:
     - schedules
   script:
+    - /opt/rir/container/install-test-deps.sh
     - cd /opt/rir/build/release
     - bin/gnur-make-tests check-recommended || $SAVE_LOGS
     - ../../tools/check-gnur-make-tests-error
@@ -123,6 +129,7 @@ tests_debug:
   except:
     - schedules
   script:
+    - /opt/rir/container/build-releaseassert.sh
     - cd /opt/rir/build/releaseassert
     - export UNSOUND_OPTS=off
     - R_ENABLE_JIT=0 ./bin/tests
@@ -136,13 +143,16 @@ tests_debug2:
   image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA
   variables:
     GIT_STRATEGY: none
+    PIR_LLVM_OPT_LEVEL: 0
+    PIR_OPT_LEVEL: 0
   stage: Run tests
   needs:
     - rir_container
   except:
     - schedules
   script:
-    - cd /opt/rir/build/releaseassert
+    - /opt/rir/container/install-test-deps.sh
+    - cd /opt/rir/build/release
     - ./bin/gnur-make-tests check-devel || $SAVE_LOGS
     - ../../tools/check-gnur-make-tests-error
   artifacts:
@@ -164,7 +174,12 @@ tests_fullverify:
   except:
     - schedules
   script:
+    - curl 10.200.14.25:8080/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz > /opt/rir/external/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz
+    - mkdir -p /opt/rir/build/fullverifier
     - cd /opt/rir/build/fullverifier
+    - /opt/rir/tools/fetch-llvm.sh
+    - cmake -DCMAKE_BUILD_TYPE=fullverifier ../..
+    - make -j6
     - bin/tests
 
 # Test particular features, like deoptimization and serialization
@@ -179,18 +194,15 @@ test_features_1:
   except:
     - schedules
   script:
-    - cd /opt/rir/build/releaseassert
-#    - PIR_ENABLE_PROFILER=1 ./bin/tests
+    - /opt/rir/container/install-test-deps.sh
     - cd /opt/rir/build/release
-    - PIR_DEOPT_CHAOS=1000 PIR_INLINER_MAX_INLINEE_SIZE=800 bin/gnur-make-tests check || $SAVE_LOGS
-    - PIR_WARMUP=2 PIR_DEOPT_CHAOS=400 ./bin/gnur-make-tests check || $SAVE_LOGS
-    - RIR_SERIALIZE_CHAOS=1 FAST_TESTS=1 ./bin/tests
-    - PIR_GLOBAL_SPECIALIZATION_LEVEL=0 ./bin/tests
-    - PIR_GLOBAL_SPECIALIZATION_LEVEL=1 ./bin/tests
-    - PIR_GLOBAL_SPECIALIZATION_LEVEL=2 ./bin/tests
-    - PIR_GLOBAL_SPECIALIZATION_LEVEL=3 ./bin/tests
-    - PIR_GLOBAL_SPECIALIZATION_LEVEL=4 ./bin/tests
-    - PIR_GLOBAL_SPECIALIZATION_LEVEL=5 ./bin/tests
+    - PIR_WARMUP=2 PIR_DEOPT_CHAOS=500 ./bin/gnur-make-tests check || $SAVE_LOGS
+    - PIR_GLOBAL_SPECIALIZATION_LEVEL=0 FAST_TESTS=1 ./bin/tests
+    - PIR_GLOBAL_SPECIALIZATION_LEVEL=1 FAST_TESTS=1 ./bin/tests
+    - PIR_GLOBAL_SPECIALIZATION_LEVEL=2 FAST_TESTS=1 ./bin/tests
+    - PIR_GLOBAL_SPECIALIZATION_LEVEL=3 FAST_TESTS=1 ./bin/tests
+    - PIR_GLOBAL_SPECIALIZATION_LEVEL=4 FAST_TESTS=1 ./bin/tests
+    - PIR_GLOBAL_SPECIALIZATION_LEVEL=5 FAST_TESTS=1 ./bin/tests
   artifacts:
     paths:
     - logs
@@ -208,6 +220,7 @@ test_features_2:
   except:
     - schedules
   script:
+    - /opt/rir/container/install-test-deps.sh
     - cd /opt/rir/build/release
     - ./bin/gnur-make-tests check || $SAVE_LOGS
   artifacts:
@@ -216,8 +229,31 @@ test_features_2:
     when: on_failure
     expire_in: 1 week
 
+test_features_3:
+  image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA
+  variables:
+    GIT_STRATEGY: none
+    PIR_LLVM_OPT_LEVEL: 0
+    PIR_OPT_LEVEL: 0
+  stage: Run tests
+  needs:
+    - rir_container
+  except:
+    - schedules
+  script:
+    - /opt/rir/container/install-test-deps.sh
+    - cd /opt/rir/build/release
+    - PIR_DEOPT_CHAOS=1000 bin/gnur-make-tests check || $SAVE_LOGS
+    - RIR_SERIALIZE_CHAOS=1 FAST_TESTS=1 ./bin/tests
+  artifacts:
+    paths:
+    - logs
+    when: on_failure
+    expire_in: 1 week
+
+
 # Run ubsan and gc torture
-test_sanitize1:
+test_gctorture1:
   image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA
   variables:
     GIT_STRATEGY: none
@@ -228,26 +264,32 @@ test_sanitize1:
   except:
     - schedules
   script:
-    - cd /opt/rir/build/releaseassert
+    - /opt/rir/container/install-test-deps.sh
+    - cd /opt/rir/build/release
     - R_GCTORTURE=5000 ./bin/gnur-make-tests check || $SAVE_LOGS
   artifacts:
     paths:
     - logs
     when: on_failure
     expire_in: 1 week
 
-test_sanitize2:
+test_gctorture2:
   image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA
   variables:
     GIT_STRATEGY: none
+    PIR_LLVM_OPT_LEVEL: 0
   stage: Run tests
   needs:
     - rir_container
   except:
     - schedules
   script:
+    - cd /opt/rir/build/release
+    - PIR_INLINER_MAX_INLINEE_SIZE=2000 ./bin/tests
+    - PIR_INLINER_MAX_INLINEE_SIZE=1500 PIR_DEOPT_CHAOS=100 ./bin/tests
+    - /opt/rir/container/build-releaseassert.sh
     - cd /opt/rir/build/releaseassert
-    - PIR_TEST_CLEAR_TEMPS=1 R_GCTORTURE=50 bin/tests
+    - PIR_TEST_CLEAR_TEMPS=1 R_GCTORTURE=60 bin/tests
 
 # Test with inlining big functions
 test_big_inline:
@@ -256,23 +298,46 @@ test_big_inline:
     GIT_STRATEGY: none
     PIR_MAX_INPUT_SIZE: 5000
     PIR_INLINER_MAX_SIZE: 5000
+    PIR_INLINER_INLINE_UNLIKELY: 1
+    PIR_INLINER_MAX_INLINEE_SIZE: 400
     PIR_LLVM_OPT_LEVEL: 0
   stage: Run tests
   needs:
     - rir_container
   except:
     - schedules
   script:
+    - /opt/rir/container/install-test-deps.sh
     - cd /opt/rir/build/release
-    - PIR_INLINER_MAX_INLINEE_SIZE=2000 ./bin/tests
-    - PIR_INLINER_MAX_INLINEE_SIZE=1500 PIR_DEOPT_CHAOS=100 ./bin/tests
-    - PIR_INLINER_MAX_INLINEE_SIZE=400 PIR_INLINER_INLINE_UNLIKELY=1 ./bin/gnur-make-tests check || $SAVE_LOGS
+    - ./bin/gnur-make-tests check || $SAVE_LOGS
   artifacts:
     paths:
     - logs
     when: on_failure
     expire_in: 1 week
 
+# Test with asan and ubsan
+test_sanitize:
+  image: registry.gitlab.com/rirvm/rir_mirror:$CI_COMMIT_SHA
+  variables:
+    GIT_STRATEGY: none
+  stage: Run tests
+  needs:
+    - rir_container
+  except:
+    - schedules
+  script:
+    - curl 10.200.14.25:8080/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz > /opt/rir/external/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz
+    - mkdir /opt/rir/build/sanitize
+    - cd /opt/rir/build/sanitize
+    - /opt/rir/tools/fetch-llvm.sh
+    - CC=$(ls ../../external/clang*/bin/clang) CXX=$(ls ../../external/clang*/bin/clang) cmake -DCMAKE_BUILD_TYPE=sanitize ../..
+    - make -j6
+    # R_LD_PRELOAD is a feature of the test-runner. To repro this without the testrunner use LD_PRELOAD instead.
+    - LSAN_OPTIONS="symbolize=1" ASAN_SYMBOLIZER_PATH=$(ls /opt/rir/external/clang*/bin/llvm-symbolizer) R_LD_PRELOAD=$(ls /opt/rir/external/clang*/lib/clang/12.0.0/lib/linux/libclang_rt.asan-x86_64.so) bin/tests
+  # sometimes leak sanitizer segfaults
+  retry: 1
+
 # Test the benchmarks container before deploying
 test_benchmarks:
   image: registry.gitlab.com/rirvm/rir_mirror/benchmark:$CI_COMMIT_SHA
@@ -342,7 +407,7 @@ benchmark_llvm:
     paths:
     - benchmarks.data
     expire_in: 24 month
-  retry: 2
+  retry: 1
 
 benchmark_gnur:
   image: registry.gitlab.com/rirvm/rir_mirror/benchmark-baseline

.vscode/c_cpp_properties.json

@@ -8,12 +8,15 @@
                 "${workspaceFolder}/external/llvm-11/include/**",
                 "${workspaceFolder}/external/llvm-11.0.1.src/include/**"
             ],
-            "defines": ["ENABLE_SLOWASSERT"],
+            "defines": [
+                "ENABLE_SLOWASSERT"
+            ],
             "compilerPath": "/usr/bin/g++-8",
             "cStandard": "c11",
             "cppStandard": "c++14",
-            "intelliSenseMode": "gcc-x64"
+            "intelliSenseMode": "gcc-x64",
+            "compileCommands": "${workspaceFolder}/build/debug/compile_commands.json"
         }
     ],
     "version": 4
-}
+}

.vscode/settings.json

@@ -112,8 +112,6 @@
     "cfenv": "cpp",
     "csignal": "cpp",
     "__functional_base_03": "cpp",
-    "__memory": "cpp",
-    "typeindex": "cpp",
-    "variant": "cpp"
+    "__memory": "cpp"
   }
 }

.ycm_extra_conf.py

@@ -14,8 +14,8 @@ def Settings( **kwargs ):
        '-DENABLE_SLOWASSERT',
        '-I'+DirectoryOfThisScript()+'/rir/src',
        '-isystem'+DirectoryOfThisScript()+'/external/custom-r/include',
-       '-isystem'+DirectoryOfThisScript()+'/external/llvm-11/include',
-       '-isystem'+DirectoryOfThisScript()+'/external/llvm-11.0.1.src/include',
+       '-isystem'+DirectoryOfThisScript()+'/external/llvm-12/include',
+       '-isystem'+DirectoryOfThisScript()+'/external/llvm-12.0.0.src/include',
        '-Wall',
        '-Wuninitialized',
        '-Wundef',

CMakeLists.txt

@@ -7,7 +7,7 @@ set(R_HOME ${CMAKE_SOURCE_DIR}/external/custom-r)
 set(R_LIBRARY_TREE ${CMAKE_SOURCE_DIR}/packages)
 set(R_ROOT_DIR ${R_HOME})
 set(R_INCLUDE_DIR ${R_HOME}/include)
-set(LLVM_DIR ${CMAKE_SOURCE_DIR}/external/llvm-11)
+set(LLVM_DIR ${CMAKE_SOURCE_DIR}/external/llvm-12)
 
 set(R_COMMAND ${R_HOME}/bin/R)
 
@@ -54,8 +54,8 @@ set(CMAKE_C_FLAGS_DEBUG "-O0 -DENABLE_SLOWASSERT")
 set(CMAKE_C_FLAGS_DEBUGOPT "-Og -DENABLE_SLOWASSERT")
 set(CMAKE_C_FLAGS "-std=gnu99")
 
-set(CMAKE_CXX_FLAGS_SANITIZE "${CMAKE_CXX_FLAGS_DEBUGOPT} -fsanitize=undefined -fno-sanitize=alignment -fsanitize-undefined-trap-on-error -fstack-protector")
-set(CMAKE_C_FLAGS_SANITIZE "${CMAKE_C_FLAGS_DEBUGOPT} -fsanitize=undefined -fno-sanitize=alignment -fsanitize-undefined-trap-on-error -fstack-protector")
+set(CMAKE_CXX_FLAGS_SANITIZE "${CMAKE_CXX_FLAGS_RELEASE} -g2 -fno-omit-frame-pointer -fsanitize=undefined -fsanitize=address -fsanitize=leak -fno-sanitize=alignment -shared-libasan -fvisibility=default")
+set(CMAKE_C_FLAGS_SANITIZE "${CMAKE_C_FLAGS_RELEASE} -g2 -fno-omit-frame-pointer -fsanitize=undefined -fsanitize=address -fsanitize=leak -fno-sanitize=alignment -shared-libasan -fvisibility=default")
 
 MARK_AS_ADVANCED(
     CMAKE_CXX_FLAGS_SANITIZE
@@ -119,11 +119,13 @@ add_custom_target(setup-build-dir
 
 if(${MACOS_USE_GCC_9})
     add_custom_target(dependencies
-        COMMAND ${CMAKE_SOURCE_DIR}/tools/sync.sh --macos_gcc9
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/build-gnur.sh --macos_gcc9
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/fetch-llvm.sh --macos_gcc9
     )
 else()
     add_custom_target(dependencies
-        COMMAND ${CMAKE_SOURCE_DIR}/tools/sync.sh
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/build-gnur.sh
+        COMMAND ${CMAKE_SOURCE_DIR}/tools/fetch-llvm.sh
     )
 endif()
 

Dockerfile

@@ -1,9 +1,20 @@
-FROM registry.gitlab.com/rirvm/rir_mirror/base
+FROM ubuntu:20.04
 ARG CI_COMMIT_SHA
 ADD . /opt/rir
-RUN echo $CI_COMMIT_SHA > /opt/rir_version && apt-get update && apt-get install -y lsb-release
-RUN cd /opt/rir && tools/sync.sh && tools/build-gnur.sh custom-r && rm -rf external/custom-r/cache_recommended.tar .git && find external -type f -name '*.o' -exec rm -f {} \; &&\
-    mkdir -p /opt/rir/build/release      && cd /opt/rir/build/release      && cmake -DCMAKE_BUILD_TYPE=release      -GNinja ../.. && ninja && bin/tests && \
-    mkdir -p /opt/rir/build/fullverifier && cd /opt/rir/build/fullverifier && cmake -DCMAKE_BUILD_TYPE=fullverifier -GNinja ../.. && ninja && \
-    mkdir -p /opt/rir/build/releaseassert && cd /opt/rir/build/releaseassert && cmake -DCMAKE_BUILD_TYPE=releaseslowassert -GNinja ../.. && ninja && \
-    rm -rf /opt/rir/external/libjit /opt/rir/external/llvm-11 /opt/rir/external/clang+llvm-* /opt/rir/external/*.tar.xz /opt/rir/build/*/CMakeFiles /opt/rir/external/custom-r/src/main
+RUN echo $CI_COMMIT_SHA > /opt/rir_version && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get upgrade -y -qq && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y -qq curl git gcc gfortran g++ libreadline-dev libx11-dev libxt-dev zlib1g-dev libbz2-dev liblzma-dev libpcre3-dev libcurl4-openssl-dev libcairo2-dev make libreadline8 libncurses-dev xz-utils cmake tcl-dev tk-dev && \
+    cd /opt/rir && \
+    tools/build-gnur.sh && \
+    rm -rf external/custom-r/cache_recommended.tar .git && \
+    find external -type f -name '*.o' -exec rm -f {} \; && \
+    apt-get clean
+RUN mkdir -p /opt/rir/build/release && \
+    cd /opt/rir && \
+    (curl 10.200.14.25:8080/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz > external/clang+llvm-12.0.0-x86_64-linux-gnu-ubuntu-20.04.tar.xz || true) && \
+    tools/fetch-llvm.sh && \
+    cd /opt/rir/build/release && \
+    cmake -DCMAKE_BUILD_TYPE=release ../.. && \
+    make -j8 && \
+    rm -rf CMakeFiles /opt/rir/external/clang+llvm*