1d99048c95
Some checks failed
CI / lint (push) Successful in 1m37s
CI / test-python (push) Successful in 1m49s
CI / test-zig (push) Successful in 1m39s
CI / test-wasm (push) Successful in 1m54s
CI / test (push) Successful in 14m44s
CI / miri (push) Successful in 14m18s
CI / build (push) Successful in 1m9s
CI / fuzz-regression (push) Successful in 9m9s
CI / publish (push) Failing after 1m10s
CI / publish-python (push) Failing after 1m46s
CI / publish-wasm (push) Has been cancelled
Signed-off-by: Kamal Tufekcic <kamal@lo.sh>
342 lines
12 KiB
Bash
Executable file
342 lines
12 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Post-run fuzzing statistics for soliton.
|
|
#
|
|
# Shows per-target and overall stats: corpus size, coverage, crashes, etc.
|
|
# If logs from fuzz_overnight.sh are available, also shows execution counts
|
|
# and throughput from the run.
|
|
# Covers both core (soliton) and CAPI (soliton_capi) fuzz targets.
|
|
#
|
|
# Usage: ./fuzz_stats.sh [--quick] [--logs <dir>]
|
|
# --quick Skip coverage measurement (no build/run, filesystem stats only)
|
|
# --logs <dir> Path to log directory (default: latest from fuzz_overnight.sh)
|
|
set -euo pipefail
|
|
|
|
QUICK=false
|
|
LOG_DIR=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--quick) QUICK=true; shift ;;
|
|
--logs) LOG_DIR="$2"; shift 2 ;;
|
|
*) echo "Unknown option: $1"; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
CORE_DIR="soliton"
|
|
CAPI_DIR="soliton_capi"
|
|
CORE_FUZZ="${CORE_DIR}/fuzz"
|
|
CAPI_FUZZ="${CAPI_DIR}/fuzz"
|
|
|
|
# Auto-find latest log directory if not specified.
|
|
if [ -z "$LOG_DIR" ] && [ -L "fuzz_logs/latest" ]; then
|
|
LOG_DIR="$(cd "fuzz_logs/latest" && pwd)"
|
|
fi
|
|
|
|
CORE_TARGETS=(
|
|
fuzz_ed25519_verify
|
|
fuzz_hybrid_verify
|
|
# fuzz_xwing_roundtrip # Excluded: 7 corpus entries after 90.5B execs, fully saturated (keygen→encap→decap, no adversarial input)
|
|
fuzz_ratchet_decrypt
|
|
fuzz_ratchet_decrypt_stateful
|
|
fuzz_ratchet_encrypt
|
|
fuzz_kex_receive_session
|
|
fuzz_kex_verify_bundle
|
|
fuzz_identity_sign_verify
|
|
# fuzz_auth_respond # Excluded: 4 corpus entries after 68B execs, fully saturated (single decap→HMAC)
|
|
fuzz_storage_decrypt_blob
|
|
# fuzz_identity_from_bytes # Excluded: 107 edges, 3 length checks only, saturates instantly
|
|
fuzz_decrypt_first_message
|
|
fuzz_storage_encrypt_blob
|
|
# fuzz_verification_phrase # Excluded: 88 edges, 29 corpus entries, trivial surface (SHA3→wordlist index)
|
|
fuzz_ratchet_roundtrip
|
|
fuzz_session_init_roundtrip
|
|
fuzz_call_derive
|
|
# fuzz_auth_verify # Excluded: 88 edges, saturated after ~15 corpus entries (single ct_eq call)
|
|
fuzz_ratchet_from_bytes_epoch
|
|
fuzz_kex_decode_receive
|
|
fuzz_dm_queue_roundtrip
|
|
fuzz_dm_queue_decrypt_blob
|
|
fuzz_argon2_params
|
|
fuzz_stream_decrypt
|
|
fuzz_stream_decrypt_at
|
|
fuzz_stream_encrypt_decrypt
|
|
fuzz_stream_encrypt_at
|
|
fuzz_ratchet_state_machine
|
|
)
|
|
|
|
CAPI_TARGETS=(
|
|
fuzz_capi_ratchet_from_bytes
|
|
fuzz_capi_storage_decrypt
|
|
fuzz_capi_decode_session_init
|
|
fuzz_capi_dm_queue_decrypt
|
|
fuzz_capi_stream_decrypt
|
|
fuzz_capi_stream_decrypt_at
|
|
fuzz_capi_stream_encrypt_at
|
|
)
|
|
|
|
HAS_LOGS=false
|
|
if [ -n "$LOG_DIR" ] && [ -d "$LOG_DIR" ]; then
|
|
HAS_LOGS=true
|
|
fi
|
|
|
|
human_size() {
|
|
local bytes=$1
|
|
if (( bytes >= 1048576 )); then
|
|
echo "$((bytes / 1048576))MB"
|
|
elif (( bytes >= 1024 )); then
|
|
echo "$((bytes / 1024))KB"
|
|
else
|
|
echo "${bytes}B"
|
|
fi
|
|
}
|
|
|
|
human_count() {
|
|
local n=$1
|
|
if (( n >= 1000000000 )); then
|
|
echo "$(awk "BEGIN {printf \"%.1fB\", $n / 1000000000}")"
|
|
elif (( n >= 1000000 )); then
|
|
echo "$(awk "BEGIN {printf \"%.1fM\", $n / 1000000}")"
|
|
elif (( n >= 1000 )); then
|
|
echo "$(awk "BEGIN {printf \"%.1fK\", $n / 1000}")"
|
|
else
|
|
echo "$n"
|
|
fi
|
|
}
|
|
|
|
# Parse a log file for libFuzzer stats.
|
|
# With -jobs=N there may be multiple stat blocks — sum executions, avg exec/s, max rss.
|
|
parse_log() {
|
|
local log_file="$1"
|
|
local stat="$2"
|
|
[ -f "$log_file" ] || { echo "0"; return; }
|
|
grep "stat::${stat}:" "$log_file" 2>/dev/null | awk -F: '{s += $NF} END {print s+0}'
|
|
}
|
|
|
|
parse_log_max() {
|
|
local log_file="$1"
|
|
local stat="$2"
|
|
[ -f "$log_file" ] || { echo "0"; return; }
|
|
grep "stat::${stat}:" "$log_file" 2>/dev/null | awk -F: '{v=$NF+0; if(v>m)m=v} END {print m+0}'
|
|
}
|
|
|
|
print_header() {
|
|
if $QUICK; then
|
|
if $HAS_LOGS; then
|
|
printf "%-32s %7s %9s %10s %9s %6s %7s %5s %4s\n" \
|
|
"Target" "Corpus" "Size" "Execs" "Exec/s" "RSS" "Crashes" "T/O" "OOM"
|
|
printf "%-32s %7s %9s %10s %9s %6s %7s %5s %4s\n" \
|
|
"---" "---" "---" "---" "---" "---" "---" "---" "---"
|
|
else
|
|
printf "%-32s %7s %9s %7s %5s %4s\n" \
|
|
"Target" "Corpus" "Size" "Crashes" "T/O" "OOM"
|
|
printf "%-32s %7s %9s %7s %5s %4s\n" \
|
|
"---" "---" "---" "---" "---" "---"
|
|
fi
|
|
else
|
|
if $HAS_LOGS; then
|
|
printf "%-32s %7s %9s %5s %8s %10s %9s %6s %7s %5s %4s\n" \
|
|
"Target" "Corpus" "Size" "Cov" "Features" "Execs" "Exec/s" "RSS" "Crashes" "T/O" "OOM"
|
|
printf "%-32s %7s %9s %5s %8s %10s %9s %6s %7s %5s %4s\n" \
|
|
"---" "---" "---" "---" "---" "---" "---" "---" "---" "---" "---"
|
|
else
|
|
printf "%-32s %7s %9s %5s %8s %7s %5s %4s\n" \
|
|
"Target" "Corpus" "Size" "Cov" "Features" "Crashes" "T/O" "OOM"
|
|
printf "%-32s %7s %9s %5s %8s %7s %5s %4s\n" \
|
|
"---" "---" "---" "---" "---" "---" "---" "---"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
total_corpus=0
|
|
total_size=0
|
|
total_crashes=0
|
|
total_timeouts=0
|
|
total_ooms=0
|
|
total_execs=0
|
|
|
|
# Process one target group. Arguments: fuzz_dir fuzz_root target...
|
|
process_targets() {
|
|
local fuzz_dir="$1"
|
|
local fuzz_root="$2"
|
|
shift 2
|
|
local targets=("$@")
|
|
|
|
for target in "${targets[@]}"; do
|
|
corpus_dir="${fuzz_root}/corpus/${target}"
|
|
artifact_dir="${fuzz_root}/artifacts/${target}"
|
|
|
|
# Corpus: file count + total bytes.
|
|
if [ -d "$corpus_dir" ]; then
|
|
corpus_count=$(find "$corpus_dir" -maxdepth 1 -type f | wc -l)
|
|
corpus_bytes=$(find "$corpus_dir" -maxdepth 1 -type f -printf '%s\n' 2>/dev/null | awk '{s+=$1}END{print s+0}')
|
|
else
|
|
corpus_count=0
|
|
corpus_bytes=0
|
|
fi
|
|
|
|
# Artifacts: crashes, timeouts, OOM (libFuzzer names them crash-*, timeout-*, oom-*).
|
|
if [ -d "$artifact_dir" ]; then
|
|
crashes=$(find "$artifact_dir" -maxdepth 1 -name 'crash-*' -type f | wc -l)
|
|
timeouts=$(find "$artifact_dir" -maxdepth 1 -name 'timeout-*' -type f | wc -l)
|
|
ooms=$(find "$artifact_dir" -maxdepth 1 -name 'oom-*' -type f | wc -l)
|
|
else
|
|
crashes=0
|
|
timeouts=0
|
|
ooms=0
|
|
fi
|
|
|
|
# Log stats (from overnight run).
|
|
execs_fmt="-"
|
|
execps_fmt="-"
|
|
rss_fmt="-"
|
|
execs_raw=0
|
|
if $HAS_LOGS; then
|
|
log_file="${LOG_DIR}/${target}.log"
|
|
if [ -f "$log_file" ]; then
|
|
execs_raw=$(parse_log "$log_file" "number_of_executed_units")
|
|
execps_raw=$(parse_log "$log_file" "average_exec_per_sec")
|
|
rss_raw=$(parse_log_max "$log_file" "peak_rss_mb")
|
|
execs_fmt=$(human_count "$execs_raw")
|
|
if (( execps_raw > 0 )); then
|
|
execps_fmt=$(human_count "$execps_raw")
|
|
else
|
|
execps_fmt="-"
|
|
fi
|
|
if (( rss_raw > 0 )); then
|
|
rss_fmt="${rss_raw}MB"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Coverage: replay corpus with -runs=0.
|
|
cov="-"
|
|
ft="-"
|
|
if ! $QUICK && [ "$corpus_count" -gt 0 ]; then
|
|
output=$(cd "$fuzz_dir" && cargo +nightly fuzz run "$target" \
|
|
"fuzz/corpus/${target}" -- -runs=0 -print_final_stats=1 -max_len=65536 2>&1 || true)
|
|
cov=$(echo "$output" | grep -oP 'cov: \K[0-9]+' | tail -1) || cov="-"
|
|
ft=$(echo "$output" | grep -oP 'ft: \K[0-9]+' | tail -1) || ft="-"
|
|
fi
|
|
|
|
size_fmt=$(human_size "$corpus_bytes")
|
|
|
|
if $QUICK; then
|
|
if $HAS_LOGS; then
|
|
printf "%-32s %7d %9s %10s %9s %6s %7d %5d %4d\n" \
|
|
"$target" "$corpus_count" "$size_fmt" "$execs_fmt" "$execps_fmt" "$rss_fmt" "$crashes" "$timeouts" "$ooms"
|
|
else
|
|
printf "%-32s %7d %9s %7d %5d %4d\n" \
|
|
"$target" "$corpus_count" "$size_fmt" "$crashes" "$timeouts" "$ooms"
|
|
fi
|
|
else
|
|
if $HAS_LOGS; then
|
|
printf "%-32s %7d %9s %5s %8s %10s %9s %6s %7d %5d %4d\n" \
|
|
"$target" "$corpus_count" "$size_fmt" "${cov:--}" "${ft:--}" "$execs_fmt" "$execps_fmt" "$rss_fmt" "$crashes" "$timeouts" "$ooms"
|
|
else
|
|
printf "%-32s %7d %9s %5s %8s %7d %5d %4d\n" \
|
|
"$target" "$corpus_count" "$size_fmt" "${cov:--}" "${ft:--}" "$crashes" "$timeouts" "$ooms"
|
|
fi
|
|
fi
|
|
|
|
total_corpus=$((total_corpus + corpus_count))
|
|
total_size=$((total_size + corpus_bytes))
|
|
total_crashes=$((total_crashes + crashes))
|
|
total_timeouts=$((total_timeouts + timeouts))
|
|
total_ooms=$((total_ooms + ooms))
|
|
total_execs=$((total_execs + execs_raw))
|
|
done
|
|
}
|
|
|
|
# Header
|
|
echo "soliton fuzzing statistics"
|
|
echo "============================"
|
|
if $HAS_LOGS; then
|
|
echo " Logs: ${LOG_DIR}"
|
|
fi
|
|
echo ""
|
|
|
|
print_header
|
|
|
|
echo ""
|
|
echo "Core (${#CORE_TARGETS[@]} targets)"
|
|
process_targets "$CORE_DIR" "$CORE_FUZZ" "${CORE_TARGETS[@]}"
|
|
|
|
echo ""
|
|
echo "CAPI (${#CAPI_TARGETS[@]} targets)"
|
|
process_targets "$CAPI_DIR" "$CAPI_FUZZ" "${CAPI_TARGETS[@]}"
|
|
|
|
# Totals
|
|
total_size_fmt=$(human_size "$total_size")
|
|
total_execs_fmt=$(human_count "$total_execs")
|
|
TOTAL_TARGETS=$(( ${#CORE_TARGETS[@]} + ${#CAPI_TARGETS[@]} ))
|
|
|
|
echo ""
|
|
if $QUICK; then
|
|
if $HAS_LOGS; then
|
|
printf "%-32s %7s %9s %10s %9s %6s %7s %5s %4s\n" \
|
|
"---" "---" "---" "---" "---" "---" "---" "---" "---"
|
|
printf "%-32s %7d %9s %10s %9s %6s %7d %5d %4d\n" \
|
|
"TOTAL (${TOTAL_TARGETS})" "$total_corpus" "$total_size_fmt" "$total_execs_fmt" "" "" "$total_crashes" "$total_timeouts" "$total_ooms"
|
|
else
|
|
printf "%-32s %7s %9s %7s %5s %4s\n" \
|
|
"---" "---" "---" "---" "---" "---"
|
|
printf "%-32s %7d %9s %7d %5d %4d\n" \
|
|
"TOTAL (${TOTAL_TARGETS})" "$total_corpus" "$total_size_fmt" "$total_crashes" "$total_timeouts" "$total_ooms"
|
|
fi
|
|
else
|
|
if $HAS_LOGS; then
|
|
printf "%-32s %7s %9s %5s %8s %10s %9s %6s %7s %5s %4s\n" \
|
|
"---" "---" "---" "---" "---" "---" "---" "---" "---" "---" "---"
|
|
printf "%-32s %7d %9s %5s %8s %10s %9s %6s %7d %5d %4d\n" \
|
|
"TOTAL (${TOTAL_TARGETS})" "$total_corpus" "$total_size_fmt" "" "" "$total_execs_fmt" "" "" "$total_crashes" "$total_timeouts" "$total_ooms"
|
|
else
|
|
printf "%-32s %7s %9s %5s %8s %7s %5s %4s\n" \
|
|
"---" "---" "---" "---" "---" "---" "---" "---"
|
|
printf "%-32s %7d %9s %5s %8s %7d %5d %4d\n" \
|
|
"TOTAL (${TOTAL_TARGETS})" "$total_corpus" "$total_size_fmt" "" "" "$total_crashes" "$total_timeouts" "$total_ooms"
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# Glossary
|
|
echo "Glossary"
|
|
echo "--------"
|
|
echo " Corpus Accumulated inputs that each trigger unique code paths."
|
|
echo " Grows over time as the fuzzer discovers new coverage."
|
|
echo " Cov Edge coverage — number of unique code branches reached."
|
|
echo " Features Fine-grained coverage (edges + hit counts + value profiles)."
|
|
echo " Higher = more behavioral diversity explored."
|
|
echo " Execs Total inputs tested during the run (from logs)."
|
|
echo " Exec/s Throughput — inputs tested per second."
|
|
echo " RSS Peak memory usage of the fuzzer process."
|
|
echo " Crashes Inputs that caused a panic, abort, or segfault."
|
|
echo " T/O Inputs that exceeded the per-input time limit."
|
|
echo " OOM Inputs that exceeded the memory limit."
|
|
|
|
# Crash details
|
|
if (( total_crashes + total_timeouts + total_ooms > 0 )); then
|
|
echo ""
|
|
echo "Artifact details"
|
|
echo "----------------"
|
|
for fuzz_root in "$CORE_FUZZ" "$CAPI_FUZZ"; do
|
|
if [ "$fuzz_root" = "$CORE_FUZZ" ]; then
|
|
targets=("${CORE_TARGETS[@]}")
|
|
else
|
|
targets=("${CAPI_TARGETS[@]}")
|
|
fi
|
|
for target in "${targets[@]}"; do
|
|
artifact_dir="${fuzz_root}/artifacts/${target}"
|
|
[ -d "$artifact_dir" ] || continue
|
|
files=$(find "$artifact_dir" -maxdepth 1 -type f 2>/dev/null)
|
|
[ -z "$files" ] && continue
|
|
echo ""
|
|
echo " ${target}:"
|
|
echo "$files" | while read -r f; do
|
|
echo " $(basename "$f") ($(wc -c < "$f") bytes)"
|
|
done
|
|
done
|
|
done
|
|
echo ""
|
|
echo "Triage: cargo +nightly fuzz tmin <target> <artifact-file>"
|
|
fi
|