initial commit

src/main.rs

@@ -0,0 +1,482 @@
+//! source2rosetta — CLI front-end. A thin clap layer over `source2rosetta::pipeline`: parse args,
+//! select the game profile, dispatch to the engine.
+
+use anyhow::{Context, Result};
+use clap::{Parser, Subcommand};
+use source2rosetta::pipeline::{
+    ClassScope, backfill_cmd, corpus_model_cmd, fold_model_cmd, load_model,
+};
+use source2rosetta::produce::{
+    ProduceArgs, SeedInputs, classify_change_cmd, filter_corpus_cmd, integration_test_cmd,
+    produce_cmd, unpack_seed,
+};
+use source2rosetta::profile;
+use std::path::PathBuf;
+
+/// The game whose profile drives lib/pawn/launch/dead-weight knobs. Adding a game = a profile const + an arm.
+#[derive(Clone, Copy, clap::ValueEnum)]
+enum Game {
+    #[value(alias = "csgo")]
+    Cs2,
+    #[value(alias = "dota")]
+    Dota2,
+}
+
+#[derive(Parser)]
+#[command(
+    name = "source2rosetta",
+    about = "Locate Source-2 engine functions across builds"
+)]
+struct Cli {
+    /// Which game's profile to use — selects lib/pawn/launch/game-key/dead-weight knobs. Source-2-generic
+    /// behavior is unaffected; only the game-specific paths read the selected profile. (Known: cs2, dota2.)
+    #[arg(long, global = true, value_enum, default_value = "cs2")]
+    game: Game,
+    #[command(subcommand)]
+    cmd: Cmd,
+}
+
+#[derive(Subcommand)]
+enum Cmd {
+    /// Own the process end-to-end for CI — no human, no mod: LAUNCH a VANILLA dedicated server for the
+    /// selected `--game`, populate it (CS2: bots on an empty deathmatch; a pawn-less game like Dota waits
+    /// for its `ready_class` proxy instead), then verify the derived gamedata against it — the schema oracle,
+    /// a semantic ptrace CALL on a live pawn (pawn games only), and (with --gamedata) a full validate-live.
+    /// (Disable metamod in the game's `gameinfo.gi` for a truly vanilla run — no hooks, clean pass/fail.)
+    IntegrationTest {
+        /// Game root (contains `bin/linuxsteamrt64/<executable>` and the content dir).
+        #[arg(long = "game-dir")]
+        game_dir: PathBuf,
+        /// Dir holding the on-disk libserver.so for the offline reference (defaults to --game).
+        #[arg(long)]
+        build: Option<PathBuf>,
+        /// Server library to derive from; defaults to the active game's server lib.
+        #[arg(long)]
+        lib: Option<String>,
+        /// Seconds to wait for the server to come up and bots to spawn alive.
+        #[arg(long, default_value_t = 60)]
+        wait: u64,
+        #[arg(long)] // default resolved from the active game profile at dispatch
+        map: Option<String>,
+        /// Number of bots to fill the server with.
+        #[arg(long, default_value_t = 9)]
+        bots: u32,
+        /// Optional gamedata json to also validate-live against the running server.
+        #[arg(long)]
+        gamedata: Option<PathBuf>,
+        /// Write the validated (kept) gamedata here (with --gamedata) — so this one command owns the
+        /// server AND persists the live-validated result, no separate validate-live needed.
+        #[arg(long)]
+        out: Option<PathBuf>,
+        /// Leave the launched server running instead of killing it after the test.
+        #[arg(long)]
+        keep: bool,
+        /// With --gamedata, also run the LIVE fuzzer against this same server for N randomized probes
+        /// (0 = off). Reuses the launched server — no separate `fuzz-live` run needed for CI.
+        #[arg(long, default_value_t = 500)]
+        fuzz_iterations: usize,
+    },
+    /// The whole per-game build in ONE in-memory command: derive → fold → (if `--game-dir` is given)
+    /// validate-live + typed netvars → fold model, writing the release set (`gamedata-`/`netvars-`/`model-`/
+    /// `manifest`) into --out-dir. No per-stage intermediate files. **Pass `--game-dir` for a full,
+    /// live-validated build; omit it for a fast OFFLINE build (gamedata + model only, no server).**
+    Produce {
+        /// A launchable game install → the FULL build (boots a server for validate-live + typed netvars).
+        /// OMIT for an offline build (gamedata + model only). The offline/full switch — no separate flag.
+        #[arg(long = "game-dir")]
+        game_dir: Option<PathBuf>,
+        /// Dir holding the on-disk libs for make-sig + live validation (defaults to --game-dir, else --target).
+        #[arg(long)]
+        build: Option<PathBuf>,
+        /// Server library to derive from; defaults to the active game's server lib.
+        #[arg(long)]
+        lib: Option<String>,
+        /// One bundled seed (catalogue + naming sections) — the release form. Replaces the loose
+        /// --catalogue/--promotable/--candidates/--full-names/--extra-offsets/--extra-sigs flags.
+        #[arg(long)]
+        seed: Option<PathBuf>,
+        /// Function catalogue (loose form; omit when using --seed).
+        #[arg(long)]
+        catalogue: Option<PathBuf>,
+        /// Corpus-signal source A: the raw build binaries to fingerprint on the fly. Exactly ONE of
+        /// --corpus / --corpus-model is required (--corpus-model is the production forward-derive path).
+        #[arg(long)]
+        corpus: Option<PathBuf>,
+        /// Corpus-signal source B: a distilled `model-<game>.json` — forward-derives from the model + only the
+        /// target binary (no corpus). Also triggers the sidecar fold (model N → N+1). See --corpus.
+        #[arg(long)]
+        corpus_model: Option<PathBuf>,
+        /// The build DIRECTORY to DERIVE gamedata from — the primary input (its libs are searched by name).
+        /// A bare `.so` path is not searched; pass the directory that contains it. REQUIRED.
+        #[arg(long)]
+        target: PathBuf,
+        /// Optional: names eligible for promotion into high_confidence (from the naming producer flow).
+        /// Omit to promote nothing — the catalogue still derives in full.
+        #[arg(long)]
+        promotable: Option<PathBuf>,
+        /// Optional: prefiltered per-address context for those names (`{"candidates": [...]}`). Omit for none.
+        #[arg(long)]
+        candidates: Option<PathBuf>,
+        /// Optional: the full-slice name universe. When set, the monolith also carries an `experimental`
+        /// tier — the least-filtered inclusion band (every name guess, graded, each with a resolvable
+        /// locator but an UNVERIFIED name).
+        #[arg(long)]
+        full_names: Option<PathBuf>,
+        /// Multilib ground-truth vtable offsets to fold as high_confidence — `{lib: [{name,class,slot}]}`
+        /// (e.g. the macOS symbol transfer). Folded directly, bypassing the candidate gate.
+        #[arg(long)]
+        extra_offsets: Option<PathBuf>,
+        /// Multilib non-virtual names to fold as sigs — `{lib: [{name,addr}]}`; `make_sig` runs per lib.
+        #[arg(long)]
+        extra_sigs: Option<PathBuf>,
+        /// Byte budget for signatures the FOLD generates (the extrapolated tiers). The derive's own
+        /// `core` sigs use a separate fixed budget — this flag does not widen those.
+        #[arg(long, default_value_t = 400)]
+        sig_cap: usize,
+        #[arg(long, default_value = "vX")]
+        version: String,
+        #[arg(long)]
+        out_dir: PathBuf,
+        /// Class scope for the sidecar model fold — must match the scope the input model was distilled with.
+        #[arg(long, value_enum, default_value = "clean")]
+        class_scope: ClassScope,
+        #[arg(long, default_value_t = 90)]
+        wait: u64,
+        #[arg(long)] // default resolved from the active game profile at dispatch
+        map: Option<String>,
+        #[arg(long, default_value_t = 9)]
+        bots: u32,
+    },
+    /// Distill the whole corpus into a shippable model (vtable-alignment hops + reference fingerprints
+    /// + slot timelines) so derivation needs only the model + the target binary, not the 86 GB corpus.
+    CorpusModel {
+        /// One bundled seed — the release form; its catalogue section is what gets distilled. Replaces the
+        /// loose --catalogue (naming sections are ignored here — the model tracks catalogue names only).
+        #[arg(long)]
+        seed: Option<PathBuf>,
+        /// Function catalogue (loose form; omit when using --seed).
+        #[arg(long)]
+        catalogue: Option<PathBuf>,
+        #[arg(long)]
+        corpus: PathBuf,
+        /// Which classes get vtable-slot hops: `clean` (every real game class — the default; enough for any
+        /// modding offset to derive model-only), `all` (also template/protobuf/NetworkVar junk), or
+        /// `catalogue` (only what the catalogue names). CI compresses the model, so on-disk size isn't shipped.
+        #[arg(long, value_enum, default_value = "clean")]
+        class_scope: ClassScope,
+        #[arg(long)]
+        out: PathBuf,
+    },
+    /// Incrementally fold ONE new build into an existing model: `model N + build → model N+1`, equal to a
+    /// full re-distill over the same builds but reading only the model + the one binary (no corpus). The
+    /// production update path — keeps the model fresh per build without re-reading history.
+    FoldModel {
+        /// The existing model N (carries the `abi_obs` window the fold re-windows).
+        #[arg(long)]
+        model: PathBuf,
+        /// One bundled seed — the release form; its catalogue section is folded. Replaces the loose --catalogue.
+        #[arg(long)]
+        seed: Option<PathBuf>,
+        /// Function catalogue (loose form; omit when using --seed). Must match the model's distill catalogue.
+        #[arg(long)]
+        catalogue: Option<PathBuf>,
+        /// The one new build dir to fold in (holds the just-updated libserver.so etc.).
+        #[arg(long)]
+        build: PathBuf,
+        /// Must match the scope the model was distilled with (`clean` default).
+        #[arg(long, value_enum, default_value = "clean")]
+        class_scope: ClassScope,
+        #[arg(long)]
+        out: PathBuf,
+    },
+    /// Back-fill cross-build history for extrapolated (T3) names: for each {name, anchor} pair, resolve
+    /// the anchor STRING uniquely in every corpus build (the same string-anchor locator as `anchor`), so
+    /// a name that was a single-build guess gains a real timeline. Reports per-name history depth +
+    /// consistency-since-first-appearance — the measure of how many T3 names graduate to first-class
+    /// (a function anchored across hundreds of builds is high-confidence regardless of its T3 origin).
+    Backfill {
+        /// Raw build binaries — needed for the string-anchor half (locating a sig/self-named function in
+        /// each historical build). Omit to run only the model-only offset half.
+        #[arg(long)]
+        corpus: Option<PathBuf>,
+        /// Distilled corpus model — its `hops` back-fill an OFFSET function's vtable-slot timeline with
+        /// NO binaries (the community-PR-of-a-vtable-method path). Omit to run only the string half.
+        #[arg(long)]
+        corpus_model: Option<PathBuf>,
+        /// Server library to derive from; defaults to the active game's server lib.
+        #[arg(long)]
+        lib: Option<String>,
+        /// JSON array of {name, tier?, anchor?, class?, slot?}: `anchor` (a distinctive string it
+        /// references — its own name for self-named) drives the string half; `class`+`slot` drive the
+        /// model-hops half.
+        #[arg(long)]
+        names: PathBuf,
+        #[arg(long)]
+        threads: Option<usize>,
+        /// Write the per-name timeline report here.
+        #[arg(long)]
+        out: Option<PathBuf>,
+    },
+    /// Classify how much a library changed between two builds — the CI branch primitive. Enumerates every
+    /// function (`.eh_frame`) in each build and compares their bodies with the position-dependent bytes
+    /// (RIP-relative displacements + near-branch targets) masked out, so the verdict is shift-invariant:
+    /// a pure layout move (bodies unchanged, addresses shifted) reads as UNCHANGED, unlike a raw byte diff.
+    /// Prints `skip` (nothing meaningful changed → no release), `normal` (an ordinary patch → re-derive) or
+    /// `shift` (a toolchain/compiler change moved ~every function's codegen at once → re-derive, and the
+    /// derive leans harder on the string-anchor/vtable recovery paths) plus the exact % of the new build's
+    /// functions whose body isn't byte-identical to the
+    /// previous build's. The thresholds are heuristic defaults — calibrate `--skip-below`/`--shift-above`
+    /// against real adjacent-vs-toolchain-jump pairs.
+    ClassifyChange {
+        /// Previous build: a `.so` file directly, or a build dir to find `--lib` under.
+        #[arg(long)]
+        prev: PathBuf,
+        /// New build: a `.so` file directly, or a build dir to find `--lib` under.
+        #[arg(long)]
+        new: PathBuf,
+        /// Server library to derive from; defaults to the active game's server lib.
+        #[arg(long)]
+        lib: Option<String>,
+        /// Extra `skip` tolerance: a changed-fraction below this also counts as `skip`. Default 0 —
+        /// only a code-IDENTICAL build (0 functions changed) skips, so any real patch re-derives. Raise
+        /// it (e.g. 0.01) to also skip changes under N%. (Calibration on 339 CS2 pairs: 311 are
+        /// code-identical, real patches touch <=6 functions / <=0.08%, the 2 toolchain jumps are 34%/53%.)
+        #[arg(long, default_value_t = 0.0)]
+        skip_below: f64,
+        /// changed-fraction at or above this = `shift`. Default 0.20 — the CS2 corpus's real patches top
+        /// out near 0.08% while its two toolchain jumps are 34%/53%, so 20% cleanly separates them with
+        /// wide margin and (unlike 40%) doesn't misclassify the 34% jump as an ordinary patch.
+        #[arg(long, default_value_t = 0.20)]
+        shift_above: f64,
+        /// Emit a machine-readable JSON object instead of the human summary.
+        #[arg(long)]
+        json: bool,
+    },
+    /// Stage-1 change-aware corpus filter: walk a game's builds chronologically, collapse runs of
+    /// code-identical builds (bodies unchanged, only relocations moved) to ONE representative, label each
+    /// surviving transition normal/shift, and segment the timeline into toolchain ERAS (cut at shifts).
+    /// Writes a selection manifest (code-distinct kept builds + era/drift each). Lossless for the per-game
+    /// facts; the distinct-build set `corpus-model` distills. Digests each build once.
+    FilterCorpus {
+        #[arg(long)]
+        corpus: PathBuf,
+        /// Server library to derive from; defaults to the active game's server lib.
+        #[arg(long)]
+        lib: Option<String>,
+        /// changed-fraction below this collapses a build as code-identical. Default 0 = only exact
+        /// code-identity collapses (any real change keeps the build code-distinct).
+        #[arg(long, default_value_t = 0.0)]
+        skip_below: f64,
+        /// changed-fraction at or above this marks a toolchain shift = an era boundary (default 0.20).
+        #[arg(long, default_value_t = 0.20)]
+        shift_above: f64,
+        #[arg(long)]
+        threads: Option<usize>,
+        /// Write the selection manifest here (JSON); prints to stdout if omitted.
+        #[arg(long)]
+        out: Option<PathBuf>,
+    },
+}
+
+/// A command's `--lib`, defaulting to the game's server library when unset.
+fn lib_or_default(prof: &profile::GameProfile, lib: Option<String>) -> String {
+    lib.unwrap_or_else(|| prof.server_lib.to_string())
+}
+
+/// Resolve the catalogue for the model commands (`corpus-model`/`fold-model`) from either a `--seed` bundle
+/// (release form) or a loose `--catalogue` file. The seed's catalogue section parses to the same functions as
+/// the loose `needed-functions.json`, so the distilled/folded model is identical either way. When a seed is
+/// given, its sections unpack under a `.seed` dir beside `out` (as `produce` does beside its out-dir).
+fn model_catalogue(
+    prof: &profile::GameProfile,
+    seed: Option<PathBuf>,
+    catalogue: Option<PathBuf>,
+    out: &std::path::Path,
+) -> Result<PathBuf> {
+    match seed {
+        Some(s) => {
+            let work = out
+                .parent()
+                .unwrap_or_else(|| std::path::Path::new("."))
+                .join(".seed");
+            Ok(unpack_seed(prof, &s, &work)?.catalogue)
+        }
+        None => catalogue.context("pass --seed <bundle> or --catalogue <file>"),
+    }
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+    // Thread the resolved profile as an explicit parameter rather than a process-wide global, so the
+    // engine stays reusable per call.
+    let profile = match cli.game {
+        Game::Cs2 => &profile::CS2,
+        Game::Dota2 => &profile::DOTA,
+    };
+    match cli.cmd {
+        Cmd::IntegrationTest {
+            game_dir,
+            build,
+            lib,
+            wait,
+            map,
+            bots,
+            gamedata,
+            out,
+            keep,
+            fuzz_iterations,
+        } => {
+            let map = map.unwrap_or_else(|| profile.default_map.to_string());
+            let lib = lib_or_default(profile, lib);
+            integration_test_cmd(
+                profile,
+                &game_dir,
+                build.as_deref(),
+                &lib,
+                wait,
+                &map,
+                bots,
+                gamedata.as_deref(),
+                out.as_deref(),
+                keep,
+                fuzz_iterations,
+            )
+        }
+        Cmd::Produce {
+            game_dir,
+            build,
+            lib,
+            seed,
+            catalogue,
+            corpus,
+            corpus_model,
+            target,
+            promotable,
+            candidates,
+            full_names,
+            extra_offsets,
+            extra_sigs,
+            sig_cap,
+            version,
+            out_dir,
+            class_scope,
+            wait,
+            map,
+            bots,
+        } => {
+            // derive inputs come from a single --seed bundle (release form) or the loose flags (dev/verify).
+            let inputs = match seed {
+                Some(s) => unpack_seed(profile, &s, &out_dir.join(".seed"))?,
+                None => SeedInputs {
+                    catalogue: catalogue.context("pass --seed <bundle> or --catalogue <file>")?,
+                    promotable,
+                    candidates,
+                    full_names,
+                    extra_offsets,
+                    extra_sigs,
+                },
+            };
+            let map = map.unwrap_or_else(|| profile.default_map.to_string());
+            let lib = lib_or_default(profile, lib);
+            produce_cmd(ProduceArgs {
+                prof: profile,
+                game: game_dir.as_deref(),
+                build: build.as_deref(),
+                lib: &lib,
+                catalogue: &inputs.catalogue,
+                corpus: corpus.as_deref(),
+                corpus_model: corpus_model.as_deref(),
+                class_scope,
+                target: &target,
+                promotable: inputs.promotable.as_deref(),
+                candidates: inputs.candidates.as_deref(),
+                full_names: inputs.full_names.as_deref(),
+                extra_offsets: inputs.extra_offsets.as_deref(),
+                extra_sigs: inputs.extra_sigs.as_deref(),
+                sig_cap,
+                version: &version,
+                out_dir: &out_dir,
+                wait,
+                map: &map,
+                bots,
+            })
+        }
+        Cmd::CorpusModel {
+            seed,
+            catalogue,
+            corpus,
+            class_scope,
+            out,
+        } => {
+            let cat = model_catalogue(profile, seed, catalogue, &out)?;
+            corpus_model_cmd(profile, &cat, &corpus, class_scope, &out)
+        }
+        Cmd::FoldModel {
+            model,
+            seed,
+            catalogue,
+            build,
+            class_scope,
+            out,
+        } => {
+            let cat = model_catalogue(profile, seed, catalogue, &out)?;
+            fold_model_cmd(
+                profile,
+                load_model(&model)?,
+                &cat,
+                &build,
+                class_scope,
+                &out,
+            )
+        }
+        Cmd::Backfill {
+            corpus,
+            corpus_model,
+            lib,
+            names,
+            threads,
+            out,
+        } => {
+            let lib = lib_or_default(profile, lib);
+            backfill_cmd(
+                profile,
+                corpus.as_deref(),
+                corpus_model.as_deref(),
+                &lib,
+                &names,
+                threads,
+                out.as_deref(),
+            )
+        }
+        Cmd::ClassifyChange {
+            prev,
+            new,
+            lib,
+            skip_below,
+            shift_above,
+            json,
+        } => {
+            let lib = lib_or_default(profile, lib);
+            classify_change_cmd(&prev, &new, &lib, skip_below, shift_above, json)
+        }
+        Cmd::FilterCorpus {
+            corpus,
+            lib,
+            skip_below,
+            shift_above,
+            threads,
+            out,
+        } => {
+            let lib = lib_or_default(profile, lib);
+            filter_corpus_cmd(
+                profile,
+                &corpus,
+                &lib,
+                skip_below,
+                shift_above,
+                out.as_deref(),
+                threads,
+            )
+        }
+    }
+}