#!/usr/bin/env bash # Post-run fuzzing statistics for soliton. # # Shows per-target and overall stats: corpus size, coverage, crashes, etc. # If logs from fuzz_overnight.sh are available, also shows execution counts # and throughput from the run. # Covers both core (soliton) and CAPI (soliton_capi) fuzz targets. # # Usage: ./fuzz_stats.sh [--quick] [--logs ] # --quick Skip coverage measurement (no build/run, filesystem stats only) # --logs Path to log directory (default: latest from fuzz_overnight.sh) set -euo pipefail QUICK=false LOG_DIR="" while [[ $# -gt 0 ]]; do case "$1" in --quick) QUICK=true; shift ;; --logs) LOG_DIR="$2"; shift 2 ;; *) echo "Unknown option: $1"; exit 1 ;; esac done CORE_DIR="soliton" CAPI_DIR="soliton_capi" CORE_FUZZ="${CORE_DIR}/fuzz" CAPI_FUZZ="${CAPI_DIR}/fuzz" # Auto-find latest log directory if not specified. if [ -z "$LOG_DIR" ] && [ -L "fuzz_logs/latest" ]; then LOG_DIR="$(cd "fuzz_logs/latest" && pwd)" fi CORE_TARGETS=( fuzz_ed25519_verify fuzz_hybrid_verify # fuzz_xwing_roundtrip # Excluded: 7 corpus entries after 90.5B execs, fully saturated (keygen→encap→decap, no adversarial input) fuzz_ratchet_decrypt fuzz_ratchet_decrypt_stateful fuzz_ratchet_encrypt fuzz_kex_receive_session fuzz_kex_verify_bundle fuzz_identity_sign_verify # fuzz_auth_respond # Excluded: 4 corpus entries after 68B execs, fully saturated (single decap→HMAC) fuzz_storage_decrypt_blob # fuzz_identity_from_bytes # Excluded: 107 edges, 3 length checks only, saturates instantly fuzz_decrypt_first_message fuzz_storage_encrypt_blob # fuzz_verification_phrase # Excluded: 88 edges, 29 corpus entries, trivial surface (SHA3→wordlist index) fuzz_ratchet_roundtrip fuzz_session_init_roundtrip fuzz_call_derive # fuzz_auth_verify # Excluded: 88 edges, saturated after ~15 corpus entries (single ct_eq call) fuzz_ratchet_from_bytes_epoch fuzz_kex_decode_receive fuzz_dm_queue_roundtrip fuzz_dm_queue_decrypt_blob fuzz_argon2_params fuzz_stream_decrypt fuzz_stream_decrypt_at fuzz_stream_encrypt_decrypt fuzz_stream_encrypt_at fuzz_ratchet_state_machine ) CAPI_TARGETS=( fuzz_capi_ratchet_from_bytes fuzz_capi_storage_decrypt fuzz_capi_decode_session_init fuzz_capi_dm_queue_decrypt fuzz_capi_stream_decrypt fuzz_capi_stream_decrypt_at fuzz_capi_stream_encrypt_at ) HAS_LOGS=false if [ -n "$LOG_DIR" ] && [ -d "$LOG_DIR" ]; then HAS_LOGS=true fi human_size() { local bytes=$1 if (( bytes >= 1048576 )); then echo "$((bytes / 1048576))MB" elif (( bytes >= 1024 )); then echo "$((bytes / 1024))KB" else echo "${bytes}B" fi } human_count() { local n=$1 if (( n >= 1000000000 )); then echo "$(awk "BEGIN {printf \"%.1fB\", $n / 1000000000}")" elif (( n >= 1000000 )); then echo "$(awk "BEGIN {printf \"%.1fM\", $n / 1000000}")" elif (( n >= 1000 )); then echo "$(awk "BEGIN {printf \"%.1fK\", $n / 1000}")" else echo "$n" fi } # Parse a log file for libFuzzer stats. # With -jobs=N there may be multiple stat blocks — sum executions, avg exec/s, max rss. parse_log() { local log_file="$1" local stat="$2" [ -f "$log_file" ] || { echo "0"; return; } grep "stat::${stat}:" "$log_file" 2>/dev/null | awk -F: '{s += $NF} END {print s+0}' } parse_log_max() { local log_file="$1" local stat="$2" [ -f "$log_file" ] || { echo "0"; return; } grep "stat::${stat}:" "$log_file" 2>/dev/null | awk -F: '{v=$NF+0; if(v>m)m=v} END {print m+0}' } print_header() { if $QUICK; then if $HAS_LOGS; then printf "%-32s %7s %9s %10s %9s %6s %7s %5s %4s\n" \ "Target" "Corpus" "Size" "Execs" "Exec/s" "RSS" "Crashes" "T/O" "OOM" printf "%-32s %7s %9s %10s %9s %6s %7s %5s %4s\n" \ "---" "---" "---" "---" "---" "---" "---" "---" "---" else printf "%-32s %7s %9s %7s %5s %4s\n" \ "Target" "Corpus" "Size" "Crashes" "T/O" "OOM" printf "%-32s %7s %9s %7s %5s %4s\n" \ "---" "---" "---" "---" "---" "---" fi else if $HAS_LOGS; then printf "%-32s %7s %9s %5s %8s %10s %9s %6s %7s %5s %4s\n" \ "Target" "Corpus" "Size" "Cov" "Features" "Execs" "Exec/s" "RSS" "Crashes" "T/O" "OOM" printf "%-32s %7s %9s %5s %8s %10s %9s %6s %7s %5s %4s\n" \ "---" "---" "---" "---" "---" "---" "---" "---" "---" "---" "---" else printf "%-32s %7s %9s %5s %8s %7s %5s %4s\n" \ "Target" "Corpus" "Size" "Cov" "Features" "Crashes" "T/O" "OOM" printf "%-32s %7s %9s %5s %8s %7s %5s %4s\n" \ "---" "---" "---" "---" "---" "---" "---" "---" fi fi } total_corpus=0 total_size=0 total_crashes=0 total_timeouts=0 total_ooms=0 total_execs=0 # Process one target group. Arguments: fuzz_dir fuzz_root target... process_targets() { local fuzz_dir="$1" local fuzz_root="$2" shift 2 local targets=("$@") for target in "${targets[@]}"; do corpus_dir="${fuzz_root}/corpus/${target}" artifact_dir="${fuzz_root}/artifacts/${target}" # Corpus: file count + total bytes. if [ -d "$corpus_dir" ]; then corpus_count=$(find "$corpus_dir" -maxdepth 1 -type f | wc -l) corpus_bytes=$(find "$corpus_dir" -maxdepth 1 -type f -printf '%s\n' 2>/dev/null | awk '{s+=$1}END{print s+0}') else corpus_count=0 corpus_bytes=0 fi # Artifacts: crashes, timeouts, OOM (libFuzzer names them crash-*, timeout-*, oom-*). if [ -d "$artifact_dir" ]; then crashes=$(find "$artifact_dir" -maxdepth 1 -name 'crash-*' -type f | wc -l) timeouts=$(find "$artifact_dir" -maxdepth 1 -name 'timeout-*' -type f | wc -l) ooms=$(find "$artifact_dir" -maxdepth 1 -name 'oom-*' -type f | wc -l) else crashes=0 timeouts=0 ooms=0 fi # Log stats (from overnight run). execs_fmt="-" execps_fmt="-" rss_fmt="-" execs_raw=0 if $HAS_LOGS; then log_file="${LOG_DIR}/${target}.log" if [ -f "$log_file" ]; then execs_raw=$(parse_log "$log_file" "number_of_executed_units") execps_raw=$(parse_log "$log_file" "average_exec_per_sec") rss_raw=$(parse_log_max "$log_file" "peak_rss_mb") execs_fmt=$(human_count "$execs_raw") if (( execps_raw > 0 )); then execps_fmt=$(human_count "$execps_raw") else execps_fmt="-" fi if (( rss_raw > 0 )); then rss_fmt="${rss_raw}MB" fi fi fi # Coverage: replay corpus with -runs=0. cov="-" ft="-" if ! $QUICK && [ "$corpus_count" -gt 0 ]; then output=$(cd "$fuzz_dir" && cargo +nightly fuzz run "$target" \ "fuzz/corpus/${target}" -- -runs=0 -print_final_stats=1 -max_len=65536 2>&1 || true) cov=$(echo "$output" | grep -oP 'cov: \K[0-9]+' | tail -1) || cov="-" ft=$(echo "$output" | grep -oP 'ft: \K[0-9]+' | tail -1) || ft="-" fi size_fmt=$(human_size "$corpus_bytes") if $QUICK; then if $HAS_LOGS; then printf "%-32s %7d %9s %10s %9s %6s %7d %5d %4d\n" \ "$target" "$corpus_count" "$size_fmt" "$execs_fmt" "$execps_fmt" "$rss_fmt" "$crashes" "$timeouts" "$ooms" else printf "%-32s %7d %9s %7d %5d %4d\n" \ "$target" "$corpus_count" "$size_fmt" "$crashes" "$timeouts" "$ooms" fi else if $HAS_LOGS; then printf "%-32s %7d %9s %5s %8s %10s %9s %6s %7d %5d %4d\n" \ "$target" "$corpus_count" "$size_fmt" "${cov:--}" "${ft:--}" "$execs_fmt" "$execps_fmt" "$rss_fmt" "$crashes" "$timeouts" "$ooms" else printf "%-32s %7d %9s %5s %8s %7d %5d %4d\n" \ "$target" "$corpus_count" "$size_fmt" "${cov:--}" "${ft:--}" "$crashes" "$timeouts" "$ooms" fi fi total_corpus=$((total_corpus + corpus_count)) total_size=$((total_size + corpus_bytes)) total_crashes=$((total_crashes + crashes)) total_timeouts=$((total_timeouts + timeouts)) total_ooms=$((total_ooms + ooms)) total_execs=$((total_execs + execs_raw)) done } # Header echo "soliton fuzzing statistics" echo "============================" if $HAS_LOGS; then echo " Logs: ${LOG_DIR}" fi echo "" print_header echo "" echo "Core (${#CORE_TARGETS[@]} targets)" process_targets "$CORE_DIR" "$CORE_FUZZ" "${CORE_TARGETS[@]}" echo "" echo "CAPI (${#CAPI_TARGETS[@]} targets)" process_targets "$CAPI_DIR" "$CAPI_FUZZ" "${CAPI_TARGETS[@]}" # Totals total_size_fmt=$(human_size "$total_size") total_execs_fmt=$(human_count "$total_execs") TOTAL_TARGETS=$(( ${#CORE_TARGETS[@]} + ${#CAPI_TARGETS[@]} )) echo "" if $QUICK; then if $HAS_LOGS; then printf "%-32s %7s %9s %10s %9s %6s %7s %5s %4s\n" \ "---" "---" "---" "---" "---" "---" "---" "---" "---" printf "%-32s %7d %9s %10s %9s %6s %7d %5d %4d\n" \ "TOTAL (${TOTAL_TARGETS})" "$total_corpus" "$total_size_fmt" "$total_execs_fmt" "" "" "$total_crashes" "$total_timeouts" "$total_ooms" else printf "%-32s %7s %9s %7s %5s %4s\n" \ "---" "---" "---" "---" "---" "---" printf "%-32s %7d %9s %7d %5d %4d\n" \ "TOTAL (${TOTAL_TARGETS})" "$total_corpus" "$total_size_fmt" "$total_crashes" "$total_timeouts" "$total_ooms" fi else if $HAS_LOGS; then printf "%-32s %7s %9s %5s %8s %10s %9s %6s %7s %5s %4s\n" \ "---" "---" "---" "---" "---" "---" "---" "---" "---" "---" "---" printf "%-32s %7d %9s %5s %8s %10s %9s %6s %7d %5d %4d\n" \ "TOTAL (${TOTAL_TARGETS})" "$total_corpus" "$total_size_fmt" "" "" "$total_execs_fmt" "" "" "$total_crashes" "$total_timeouts" "$total_ooms" else printf "%-32s %7s %9s %5s %8s %7s %5s %4s\n" \ "---" "---" "---" "---" "---" "---" "---" "---" printf "%-32s %7d %9s %5s %8s %7d %5d %4d\n" \ "TOTAL (${TOTAL_TARGETS})" "$total_corpus" "$total_size_fmt" "" "" "$total_crashes" "$total_timeouts" "$total_ooms" fi fi echo "" # Glossary echo "Glossary" echo "--------" echo " Corpus Accumulated inputs that each trigger unique code paths." echo " Grows over time as the fuzzer discovers new coverage." echo " Cov Edge coverage — number of unique code branches reached." echo " Features Fine-grained coverage (edges + hit counts + value profiles)." echo " Higher = more behavioral diversity explored." echo " Execs Total inputs tested during the run (from logs)." echo " Exec/s Throughput — inputs tested per second." echo " RSS Peak memory usage of the fuzzer process." echo " Crashes Inputs that caused a panic, abort, or segfault." echo " T/O Inputs that exceeded the per-input time limit." echo " OOM Inputs that exceeded the memory limit." # Crash details if (( total_crashes + total_timeouts + total_ooms > 0 )); then echo "" echo "Artifact details" echo "----------------" for fuzz_root in "$CORE_FUZZ" "$CAPI_FUZZ"; do if [ "$fuzz_root" = "$CORE_FUZZ" ]; then targets=("${CORE_TARGETS[@]}") else targets=("${CAPI_TARGETS[@]}") fi for target in "${targets[@]}"; do artifact_dir="${fuzz_root}/artifacts/${target}" [ -d "$artifact_dir" ] || continue files=$(find "$artifact_dir" -maxdepth 1 -type f 2>/dev/null) [ -z "$files" ] && continue echo "" echo " ${target}:" echo "$files" | while read -r f; do echo " $(basename "$f") ($(wc -c < "$f") bytes)" done done done echo "" echo "Triage: cargo +nightly fuzz tmin " fi