analyze: add scripts for computing pointwise metrics

c2rust-analyze/scripts/pointwise_metrics.py

@@ -0,0 +1,75 @@
+'''
+Process logs to compute pointwise success rate metrics.
+
+These metrics are measured as follows.  For each function, we run the static
+analysis and rewrite that function in isolation, producing a new `.rs` file
+where that function has been rewritten but all other code remains the same.
+Then we remove the `unsafe` qualifier from the target function and try to
+compile the code.  The "pointwise success rate" is the number of functions on
+which this procedure succeeds.
+
+As a performance optimization, instead of running analysis separately for each
+function, we run `c2rust-analyze` with `--rewrite-mode pointwise`, which runs
+the analysis part once and then rewrites each function in isolation using the
+same analysis results.  This provides a significant speedup for large codebases
+where the static analysis portion is very slow.
+
+To provide a basis for comparison, in addition to attempting to compile all
+pointwise rewrites, we also try removing `unsafe` and compiling each function
+in the original, unmodified code.  This provides a baseline for how many
+functions are "trivially safe" without rewriting.
+'''
+
+from pprint import pprint
+import re
+import sys
+
+# `pointwise_log_path` should be a log generated by running
+# `pointwise_try_build.sh` on each output file of a pointwise rewrite
+# (`foo.*.rs`, one per function).  The outputs for all files should be
+# concatenated in a single log.  This gives the results of pointwise rewriting
+# and compiling each function.
+#
+# `unmodified_log_path` should come from `pointwise_try_build_unmodified.sh`
+# instead.  This gives results of pointwise compiling each function without
+# rewriting.
+pointwise_log_path, unmodified_log_path = sys.argv[1:]
+
+
+FUNC_ERRORS_RE = re.compile(r'^got ([0-9]+) errors for ([^ \n]+)$')
+
+def read_func_errors(f):
+    func_errors = {}
+    for line in f:
+        m = FUNC_ERRORS_RE.match(line)
+        if m is None:
+            continue
+        func = m.group(2)
+        errors = int(m.group(1))
+        assert func not in func_errors, 'duplicate entry for %r' % func
+        func_errors[func] = errors
+    return func_errors
+
+pointwise_func_errors = read_func_errors(open(pointwise_log_path))
+pointwise_ok = set(func for func, errors in pointwise_func_errors.items() if errors == 0)
+print('pointwise:  %5d/%d functions passed (%.1f%%)' % (
+    len(pointwise_ok), len(pointwise_func_errors),
+    len(pointwise_ok) / len(pointwise_func_errors) * 100))
+
+unmodified_func_errors = read_func_errors(open(unmodified_log_path))
+unmodified_ok = set(func for func, errors in unmodified_func_errors.items() if errors == 0)
+print('unmodified: %5d/%d functions passed (%.1f%%)' % (
+    len(unmodified_ok), len(unmodified_func_errors),
+    len(unmodified_ok) / len(unmodified_func_errors) * 100))
+
+assert len(pointwise_func_errors) == len(unmodified_func_errors)
+num_total = len(pointwise_func_errors)
+num_unmodified_ok = len(unmodified_ok)
+num_unmodified_bad = num_total - num_unmodified_ok
+
+improved = pointwise_ok - unmodified_ok
+print('improved:   %5d/%d functions (%.1f%%)' % (
+    len(improved), num_unmodified_bad, len(improved) / num_unmodified_bad * 100))
+broke = unmodified_ok - pointwise_ok
+print('broke:      %5d/%d functions (%.1f%%)' % (
+    len(broke), num_unmodified_ok, len(broke) / num_unmodified_ok * 100))

c2rust-analyze/scripts/pointwise_try_build.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -euo pipefail
+
+echo
+
+f=$1
+mode=$2
+shift 2
+flags=( "$@" )
+echo "f=$f"
+echo "mode=$mode"
+
+name=${f%%.*.rs}
+name=${name##**/}
+echo "name=$name"
+
+func=${f%.rs}
+func=${func##*.}
+echo "func=$func"
+
+filter_errors() {
+    jq 'select(.level == "error") | .message' -r |
+        { grep -v -e '^aborting due to ' -e '^call to unsafe function is unsafe ' || true; }
+}
+
+case "$mode" in
+    pointwise)
+        sed -i -e "/fn $func\\>/s/\\<unsafe //" $f
+        ;;
+    unmodified)
+        d="$(dirname "$f")"
+        f="$d/${name}_safe_${func}.rs"
+        cp "$d/$name.rs" "$f"
+        sed -i -e "/fn $func\\>/s/\\<unsafe //" $f
+        ;;
+    *)
+        echo "unsupported mode $mode" 1>&2
+        exit 1
+        ;;
+esac
+
+rustc --error-format json --emit metadata --crate-name $name "$f" "${flags[@]}" 2>rustc-$func.json || true
+num_lines="$(cat rustc-$func.json | filter_errors | wc -l)"
+echo "got $num_lines errors for $func"
+if [[ "$num_lines" -eq 0 ]]; then
+    exit 0
+else
+    cat rustc-$func.json | filter_errors
+    exit 1
+fi

c2rust-analyze/scripts/run_pointwise_metrics.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+set -euo pipefail
+
+# Run pointwise metrics on lighttpd_rust_amalgamated.
+
+if [[ $# -ne 1 ]]; then
+  echo "Usage: $0 <path/to/lighttpd_rust_amalgamated/>"
+  exit 1
+fi
+
+SCRIPT_DIR="$(dirname "$0")"
+
+# Get the path to lighttpd_rust_amalgamated
+MODULE_DIR="$1"
+shift 1
+
+# Find the sysroot directory of rustc
+SYSROOT="$(rustc --print sysroot)"
+
+# Find the necessary rlibs
+extern() {
+  local name=$1
+  local rlib=$(find "$MODULE_DIR/target/debug/deps" -name "lib${name}*.rlib" -print -quit)
+  echo >&2 "found rlib for $name: $rlib"
+  echo --extern $name=$rlib
+}
+
+now=$(date +%Y%m%d-%H%M%S)
+
+
+# Set $rustc_flags and run the analysis as appropriate for the target project.
+# $rustc_flags is also used below for `pointwise_try_build.sh`.
+project="$(basename "$MODULE_DIR")"
+case "$project" in
+    lighttpd_*)
+        rustc_flags=(
+            --edition 2021
+            --crate-type rlib
+            #--sysroot "$SYSROOT"
+            -L "dependency=$MODULE_DIR/target/debug/deps"
+            $(extern c2rust_bitfields)
+            $(extern libc)
+            -A warnings
+        )
+
+        C2RUST_ANALYZE_NO_CARGO=1 \
+        C2RUST_ANALYZE_REWRITE_MODE=pointwise \
+        C2RUST_ANALYZE_USE_MANUAL_SHIMS=1 \
+        cargo run --bin c2rust-analyze --release -- "$MODULE_DIR/src/main.rs" \
+            --crate-name "$(basename "$MODULE_DIR")" \
+            "${rustc_flags[@]}" \
+            |& tee pointwise-lighttpd-analyze-$now.log \
+            || true
+
+        ;;
+
+    cfs_*)
+        : cargo run --bin c2rust-analyze --release -- \
+            --rewrite-mode pointwise --use-manual-shims -- \
+            build --manifest-path "$MODULE_DIR/Cargo.toml" \
+            |& tee pointwise-cfs-analyze-$now.log \
+            || true
+
+        rustc_flags=(
+            --edition 2021
+            --crate-type rlib
+            #--sysroot "$SYSROOT"
+            -L "dependency=$MODULE_DIR/target/debug/deps"
+            $(extern c2rust_bitfields)
+            $(extern f128)
+            $(extern libc)
+            $(extern memoffset)
+            -A warnings
+        )
+
+        ;;
+
+    *)
+        echo "unsupported project $project" 1>&2
+        exit 1
+esac
+
+
+# Try to compile each function separately.
+
+pointwise_log_file=pointwise-lighttpd-pointwise-$now.log
+for f in "$MODULE_DIR"/src/main.*.rs; do
+    "$SCRIPT_DIR/pointwise_try_build.sh" "$f" pointwise "${rustc_flags[@]}" || true
+done |& tee "$pointwise_log_file"
+
+unmodified_log_file=pointwise-lighttpd-unmodified-$now.log
+for f in "$MODULE_DIR"/src/main.*.rs; do
+    "$SCRIPT_DIR/pointwise_try_build.sh" "$f" unmodified "${rustc_flags[@]}" || true
+done |& tee "$unmodified_log_file"
+
+echo
+echo 
+
+python3 "$SCRIPT_DIR/pointwise_metrics.py" "$pointwise_log_file" "$unmodified_log_file"