libsoliton/soliton/fuzz/fuzz_targets/fuzz_ratchet_state_machine.rs

#![no_main]
use libfuzzer_sys::fuzz_target;
use soliton::error::Error;
use soliton::primitives::{random, xwing};
use soliton::ratchet::RatchetState;
use std::collections::HashSet;

// Action-sequence fuzzer for the ratchet state machine.
//
// Keeps Alice and Bob as live cryptographic states and uses fuzz input to
// drive a sequence of protocol operations — sends, deliveries (possibly
// out-of-order), direction changes, and replays. Because the harness
// performs real crypto (generating valid AEAD tags), the fuzzer is not
// blocked by AEAD authentication barriers and can freely explore recv_seen
// boundaries, duplicate detection, KEM ratchet transitions, previous-epoch
// grace periods, rollback, and counter exhaustion.
//
// Unlike crash-only fuzz targets, this harness asserts protocol correctness:
// - If decrypt returns Ok, the message must not have been previously delivered
// - If decrypt returns DuplicateMessage, the message must have been previously delivered
// - Decrypted plaintext must match the original
//
// Input format: each byte is an action opcode.
//   0x00..0x3F  Alice sends (plaintext = &[opcode])
//   0x40..0x7F  Bob sends (plaintext = &[opcode])
//   0x80..0xBF  Deliver from Alice's outbox to Bob (index = opcode & 0x3F)
//   0xC0..0xFF  Deliver from Bob's outbox to Alice (index = opcode & 0x3F)
fuzz_target!(|data: &[u8]| {
    if data.len() > 200 { return; }

    let fp_a = [0xAA_u8; 32];
    let fp_b = [0xBB_u8; 32];
    let rk: [u8; 32] = random::random_array();
    let ck: [u8; 32] = random::random_array();

    let (ek_pk, ek_sk) = xwing::keygen().unwrap();
    let mut alice = RatchetState::init_alice(
        rk, ck, fp_a, fp_b, ek_pk.clone(), ek_sk,
    ).unwrap();
    let mut bob = RatchetState::init_bob(rk, ck, fp_b, fp_a, ek_pk).unwrap();

    // Outboxes hold (header, ciphertext, original_plaintext) triples.
    let mut alice_outbox: Vec<(soliton::ratchet::RatchetHeader, Vec<u8>, Vec<u8>)> = Vec::new();
    let mut bob_outbox: Vec<(soliton::ratchet::RatchetHeader, Vec<u8>, Vec<u8>)> = Vec::new();

    // Track which outbox indices have been successfully delivered.
    // A second delivery of the same index must return DuplicateMessage (or
    // AeadFailed if the previous-epoch grace window expired).
    let mut bob_delivered: HashSet<usize> = HashSet::new();
    let mut alice_delivered: HashSet<usize> = HashSet::new();

    for &opcode in data {
        match opcode {
            // Alice sends
            0x00..=0x3F => {
                let pt = vec![opcode];
                match alice.encrypt(&pt) {
                    Ok(enc) => alice_outbox.push((enc.header, enc.ciphertext, pt)),
                    Err(_) => {} // ChainExhausted, dead session — expected
                }
            }
            // Bob sends
            0x40..=0x7F => {
                let pt = vec![opcode];
                match bob.encrypt(&pt) {
                    Ok(enc) => bob_outbox.push((enc.header, enc.ciphertext, pt)),
                    Err(_) => {}
                }
            }
            // Deliver Alice's message to Bob
            0x80..=0xBF => {
                let idx = (opcode & 0x3F) as usize;
                if let Some((header, ct, expected_pt)) = alice_outbox.get(idx) {
                    match bob.decrypt(header, ct) {
                        Ok(pt) => {
                            // First successful delivery — plaintext must match.
                            assert_eq!(
                                pt.as_slice(), expected_pt.as_slice(),
                                "plaintext mismatch at alice_outbox[{idx}]"
                            );
                            // Must not have been delivered before.
                            assert!(
                                bob_delivered.insert(idx),
                                "decrypt returned Ok for already-delivered alice_outbox[{idx}]"
                            );
                        }
                        Err(Error::DuplicateMessage) => {
                            // Duplicate — must have been delivered before.
                            assert!(
                                bob_delivered.contains(&idx),
                                "DuplicateMessage for never-delivered alice_outbox[{idx}]"
                            );
                        }
                        Err(Error::AeadFailed) => {
                            // Valid if the previous-epoch grace window expired
                            // (two KEM ratchet steps since this message's epoch).
                            // Also valid for genuinely corrupted state. Not assertable.
                        }
                        Err(Error::ChainExhausted) => {
                            // recv_seen cap reached — valid.
                        }
                        Err(Error::InvalidData) => {
                            // Dead session (root_key zeroed) — valid after
                            // a prior session-fatal error.
                        }
                        Err(e) => {
                            panic!("unexpected error delivering alice_outbox[{idx}] to bob: {e:?}");
                        }
                    }
                }
            }
            // Deliver Bob's message to Alice
            0xC0..=0xFF => {
                let idx = (opcode & 0x3F) as usize;
                if let Some((header, ct, expected_pt)) = bob_outbox.get(idx) {
                    match alice.decrypt(header, ct) {
                        Ok(pt) => {
                            assert_eq!(
                                pt.as_slice(), expected_pt.as_slice(),
                                "plaintext mismatch at bob_outbox[{idx}]"
                            );
                            assert!(
                                alice_delivered.insert(idx),
                                "decrypt returned Ok for already-delivered bob_outbox[{idx}]"
                            );
                        }
                        Err(Error::DuplicateMessage) => {
                            assert!(
                                alice_delivered.contains(&idx),
                                "DuplicateMessage for never-delivered bob_outbox[{idx}]"
                            );
                        }
                        Err(Error::AeadFailed | Error::ChainExhausted | Error::InvalidData) => {}
                        Err(e) => {
                            panic!("unexpected error delivering bob_outbox[{idx}] to alice: {e:?}");
                        }
                    }
                }
            }
        }
    }
});