source2rosetta/src/main.rs
Kamal Tufekcic 22ab973f0c
All checks were successful
CI / lint (push) Successful in 16s
CI / fuzz (push) Successful in 2m3s
CI / test (push) Successful in 25s
minor tweaks
2026-08-03 03:59:53 +03:00

569 lines
26 KiB
Rust
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

//! source2rosetta — CLI front-end. A thin clap layer over BOTH engine halves —
//! `source2rosetta::pipeline` (the offline derivation engine) and `source2rosetta::produce` (CI
//! orchestration plus everything that drives a running server): parse args, select the game profile,
//! dispatch.
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use source2rosetta::pipeline::{
ClassScope, backfill_cmd, corpus_model_cmd, fold_model_cmd, load_model,
};
use source2rosetta::produce::{
ProduceArgs, SeedInputs, classify_change_cmd, filter_corpus_cmd, integration_test_cmd,
produce_cmd, unpack_seed,
};
use source2rosetta::profile;
use std::path::PathBuf;
/// The game whose profile drives lib/pawn/launch/dead-weight knobs. Adding a game = a profile const + an arm.
#[derive(Clone, Copy, clap::ValueEnum)]
enum Game {
#[value(alias = "csgo")]
Cs2,
#[value(alias = "dota")]
Dota2,
}
#[derive(Parser)]
#[command(
name = "source2rosetta",
about = "Locate Source-2 engine functions across builds"
)]
struct Cli {
/// Which game's profile to use — selects lib/pawn/launch/game-key/dead-weight knobs. Source-2-generic
/// behavior is unaffected; only the game-specific paths read the selected profile. (Known: cs2, dota2.)
#[arg(long, global = true, value_enum, default_value = "cs2")]
game: Game,
#[command(subcommand)]
cmd: Cmd,
}
#[derive(Subcommand)]
enum Cmd {
/// Own the process end-to-end for CI — no human, no mod: LAUNCH a VANILLA dedicated server for the
/// selected `--game`, populate it (CS2: bots on an empty deathmatch; a pawn-less game like Dota waits
/// for its `ready_class` proxy instead), then verify the derived gamedata against it — the schema oracle,
/// a semantic ptrace CALL on a live pawn (pawn games only), and (with --gamedata) a full validate-live.
/// (Disable metamod in the game's `gameinfo.gi` for a truly vanilla run — no hooks, clean pass/fail.)
IntegrationTest {
/// Game root (contains `bin/linuxsteamrt64/<executable>` and the content dir).
#[arg(long = "game-dir")]
game_dir: PathBuf,
/// Dir holding the on-disk libserver.so for the offline reference (defaults to --game).
#[arg(long)]
build: Option<PathBuf>,
/// Server library to derive from; defaults to the active game's server lib.
#[arg(long)]
lib: Option<String>,
/// Seconds to wait for the server to come up and reach its readiness anchor — an alive bot pawn
/// for a pawn game, a live `ready_class` instance otherwise.
#[arg(long, default_value_t = 60)]
wait: u64,
#[arg(long)] // default resolved from the active game profile at dispatch
map: Option<String>,
/// Number of bots to fill the server with. A pawn-less game uses this only to size `-maxplayers`;
/// nothing waits for a bot pawn there.
#[arg(long, default_value_t = 9)]
bots: u32,
/// Optional gamedata json to also validate-live against the running server.
#[arg(long)]
gamedata: Option<PathBuf>,
/// Write the validated (kept) gamedata here (with --gamedata) — so this one command owns the
/// server AND persists the live-validated result, no separate validate-live needed.
#[arg(long, requires = "gamedata")]
out: Option<PathBuf>,
/// Leave the launched server running instead of killing it after the test.
#[arg(long)]
keep: bool,
/// With --gamedata, also run the LIVE fuzzer against this same server for N randomized probes
/// (0 = off). PAWN GAMES ONLY — a pawn-less game runs no live fuzz. Runs against the server THIS
/// command launched: `integration-test` boots its own and does not attach to one `produce` left
/// behind.
#[arg(long, default_value_t = 500)]
fuzz_iterations: usize,
},
/// The whole per-game build in ONE in-memory command: derive → fold → (if `--game-dir` is given)
/// validate-live + typed netvars → merge → fold model, writing the release set
/// (`rosetta-<game>.json` + `manifest.json`, plus `model-<game>.json` when `--corpus-model` was the
/// source — the sidecar is that model rolled N → N+1, so a `--corpus` genesis run writes two files,
/// not three) into --out-dir. No per-stage intermediate files. **Pass `--game-dir` for a full,
/// live-validated build; omit it for a fast OFFLINE build (no server, so no live validation and a
/// `null` schema).**
Produce {
/// A launchable game install → the FULL build (boots a server for validate-live + typed netvars).
/// OMIT for an offline build. The offline/full switch — no separate flag.
#[arg(long = "game-dir")]
game_dir: Option<PathBuf>,
/// Dir holding the on-disk libs for make-sig + live validation (defaults to --game-dir, else --target).
#[arg(long)]
build: Option<PathBuf>,
/// Server library to derive from; defaults to the active game's server lib.
#[arg(long)]
lib: Option<String>,
/// One bundled seed (catalogue + naming sections) — the release form. Carries everything the loose
/// --catalogue/--promotable/--candidates/--full-names/--extra-offsets/--extra-sigs flags carry, and
/// CONFLICTS with each of them: pass one form or the other, never a mix.
#[arg(long)]
seed: Option<PathBuf>,
/// Function catalogue (loose form; omit when using --seed).
#[arg(long, conflicts_with = "seed")]
catalogue: Option<PathBuf>,
/// Corpus-signal source A: the raw build binaries to fingerprint on the fly. Exactly ONE of
/// --corpus / --corpus-model is required (--corpus-model is the production forward-derive path).
#[arg(long)]
corpus: Option<PathBuf>,
/// Corpus-signal source B: a distilled `model-<game>.json` — forward-derives from the model + only the
/// target binary (no corpus). Also triggers the sidecar fold (model N → N+1). See --corpus.
#[arg(long, conflicts_with = "corpus")]
corpus_model: Option<PathBuf>,
/// The build DIRECTORY to DERIVE gamedata from — the primary input (its libs are searched by name).
/// A bare `.so` path is not searched; pass the directory that contains it. REQUIRED.
#[arg(long)]
target: PathBuf,
/// Optional: names eligible for promotion into high_confidence (from the naming producer flow).
/// Omit to promote nothing — the catalogue still derives in full.
#[arg(long, conflicts_with = "seed")]
promotable: Option<PathBuf>,
/// Optional: prefiltered per-address context for those names (`{"candidates": [...]}`). Omit for none.
#[arg(long, conflicts_with = "seed")]
candidates: Option<PathBuf>,
/// Optional: the full-slice name universe. When set, the monolith also carries an `experimental`
/// tier — the least-filtered inclusion band (every name guess, graded, each with a resolvable
/// locator but an UNVERIFIED name).
#[arg(long, conflicts_with = "seed")]
full_names: Option<PathBuf>,
/// Multilib ground-truth vtable offsets to fold as high_confidence — `{lib: [{name,class,slot}]}`
/// (e.g. the macOS symbol transfer). Folded directly, bypassing the candidate gate.
#[arg(long, conflicts_with = "seed")]
extra_offsets: Option<PathBuf>,
/// Multilib non-virtual names to fold as sigs — `{lib: [{name,addr}]}`; `make_sig` runs per lib.
#[arg(long, conflicts_with = "seed")]
extra_sigs: Option<PathBuf>,
/// Declared C++ prototypes (`mappings/prototypes.json`) to judge against this build's measured
/// register footprints. Static repo input — omit and no function carries a declared prototype.
#[arg(long)]
prototypes: Option<PathBuf>,
/// Authored function descriptions (`mappings/semantics-<game>.json`), folded in beside each
/// function. Static repo input, keyed on the NAME — omit and no function carries one.
#[arg(long)]
semantics: Option<PathBuf>,
/// Valve's naming for the entity class behind each `PVAL_EHANDLE` Pulse parameter
/// (`mappings/ehandle-classes.json`), propagated across the parameters this build's destructor
/// addresses prove are the same type. Static repo input — omit and the bindings artifact simply
/// states no class.
#[arg(long)]
ehandle_classes: Option<PathBuf>,
/// Byte budget for signatures the FOLD generates (the extrapolated tiers). The derive's own
/// `core` sigs use a separate fixed budget — this flag does not widen those.
#[arg(long, default_value_t = 400)]
sig_cap: usize,
#[arg(long, default_value = "vX")]
version: String,
#[arg(long)]
out_dir: PathBuf,
/// Class scope for the sidecar model fold — must match the scope the input model was distilled with.
#[arg(long, value_enum, default_value = "clean")]
class_scope: ClassScope,
#[arg(long, default_value_t = 90)]
wait: u64,
#[arg(long)] // default resolved from the active game profile at dispatch
map: Option<String>,
#[arg(long, default_value_t = 9)]
bots: u32,
},
/// Distill the whole corpus into a shippable model (vtable-alignment hops + reference fingerprints
/// + slot timelines) so derivation needs only the model + the target binary, not the 86 GB corpus.
CorpusModel {
/// One bundled seed — the release form; its catalogue section is what gets distilled. CONFLICTS with
/// the loose --catalogue (naming sections are ignored here — the model tracks catalogue names only).
#[arg(long)]
seed: Option<PathBuf>,
/// Function catalogue (loose form; omit when using --seed).
#[arg(long, conflicts_with = "seed")]
catalogue: Option<PathBuf>,
#[arg(long)]
corpus: PathBuf,
/// Which classes get vtable-slot hops: `clean` (every real game class — the default; enough for any
/// modding offset to derive model-only), `all` (also template/protobuf/NetworkVar junk), or
/// `catalogue` (only what the catalogue names). CI compresses the model, so on-disk size isn't shipped.
#[arg(long, value_enum, default_value = "clean")]
class_scope: ClassScope,
#[arg(long)]
out: PathBuf,
},
/// Incrementally fold ONE new build into an existing model: `model N + build → model N+1`, equal to a
/// full re-distill over the same builds but reading only the model + the one binary (no corpus). The
/// production update path — keeps the model fresh per build without re-reading history.
FoldModel {
/// The existing model N (carries the `abi_obs` window the fold re-windows).
#[arg(long)]
model: PathBuf,
/// One bundled seed — the release form; its catalogue section is folded. CONFLICTS with the loose
/// --catalogue.
#[arg(long)]
seed: Option<PathBuf>,
/// Function catalogue (loose form; omit when using --seed). Must match the model's distill catalogue.
#[arg(long, conflicts_with = "seed")]
catalogue: Option<PathBuf>,
/// The one new build dir to fold in (holds the just-updated libserver.so etc.).
#[arg(long)]
build: PathBuf,
/// Must match the scope the model was distilled with (`clean` default).
#[arg(long, value_enum, default_value = "clean")]
class_scope: ClassScope,
#[arg(long)]
out: PathBuf,
},
/// Back-fill cross-build history for extrapolated (T3) names: for each {name, anchor} pair, resolve
/// the anchor STRING uniquely in every corpus build (the same string-anchor locator as `anchor`), so
/// a name that was a single-build guess gains a real timeline. Reports per-name history depth +
/// consistency-since-first-appearance — the measure of how many T3 names graduate to first-class
/// (a function anchored across hundreds of builds is high-confidence regardless of its T3 origin).
Backfill {
/// Raw build binaries — needed for the string-anchor half (locating a sig/self-named function in
/// each historical build). Omit to run only the model-only offset half.
#[arg(long)]
corpus: Option<PathBuf>,
/// Distilled corpus model — its `hops` back-fill an OFFSET function's vtable-slot timeline with
/// NO binaries (the community-PR-of-a-vtable-method path). Omit to run only the string half.
#[arg(long)]
corpus_model: Option<PathBuf>,
/// Server library to derive from; defaults to the active game's server lib.
#[arg(long)]
lib: Option<String>,
/// JSON array of {name, tier?, anchor?, class?, slot?}: `anchor` (a distinctive string it
/// references — its own name for self-named) drives the string half; `class`+`slot` drive the
/// model-hops half.
#[arg(long)]
names: PathBuf,
#[arg(long)]
threads: Option<usize>,
/// Write the per-name timeline report here.
#[arg(long)]
out: Option<PathBuf>,
},
/// Classify how much a library changed between two builds — the CI branch primitive. Enumerates every
/// function in each build (relocation code-pointers decoded call targets `.eh_frame` starts —
/// the FDE list alone covers ~12% of these binaries) and compares their bodies with the position-dependent bytes
/// (RIP-relative displacements + near-branch targets) masked out, so the verdict is shift-invariant:
/// a pure layout move (bodies unchanged, addresses shifted) reads as UNCHANGED, unlike a raw byte diff.
/// Prints `skip` (nothing meaningful changed → no release), `normal` (an ordinary patch → re-derive) or
/// `shift` (a toolchain/compiler change moved ~every function's codegen at once → re-derive, and the
/// derive leans harder on the string-anchor/vtable recovery paths) plus the exact % of the new build's
/// functions whose body isn't byte-identical to the
/// previous build's. The thresholds are heuristic defaults — calibrate `--skip-below`/`--shift-above`
/// against real adjacent-vs-toolchain-jump pairs.
ClassifyChange {
/// Previous build: a `.so` file directly, or a build dir to find `--lib` under.
#[arg(long)]
prev: PathBuf,
/// New build: a `.so` file directly, or a build dir to find `--lib` under.
#[arg(long)]
new: PathBuf,
/// Server library to derive from; defaults to the active game's server lib.
#[arg(long)]
lib: Option<String>,
/// Extra `skip` tolerance: a changed-fraction below this also counts as `skip`. Default 0 —
/// only a code-IDENTICAL build (0 functions changed) skips, so any real patch re-derives. Raise
/// it (e.g. 0.01) to also skip changes under N%. The default is the one setting that does not
/// depend on the calibration below: zero changed functions is zero at any denominator.
#[arg(long, default_value_t = 0.0)]
skip_below: f64,
/// changed-fraction at or above this = `shift`. Default 0.20. Measured over 344 CS2 builds
/// (~70,300 functions each): 82 are code-identical, the 252 ordinary patches run from 0.001% to
/// 17.8% (median 0.12%), and the 9 toolchain jumps start at 22.4% and reach 93.8%. 0.20 sits in
/// that gap — but the gap is ~4.6 points wide, not the wide margin an earlier calibration
/// claimed, so recalibrate before trusting `shift` on another game or a re-cut corpus.
#[arg(long, default_value_t = 0.20)]
shift_above: f64,
/// Emit a machine-readable JSON object instead of the human summary.
#[arg(long)]
json: bool,
},
/// Stage-1 change-aware corpus filter: walk a game's builds chronologically, collapse runs of
/// code-identical builds (bodies unchanged, only relocations moved) to ONE representative, label each
/// surviving transition normal/shift, and segment the timeline into toolchain ERAS (cut at shifts).
/// Writes a selection manifest (code-distinct kept builds + era/drift each). Lossless for the per-game
/// facts; the distinct-build set `corpus-model` distills. Digests each build once.
FilterCorpus {
#[arg(long)]
corpus: PathBuf,
/// Server library to derive from; defaults to the active game's server lib.
#[arg(long)]
lib: Option<String>,
/// changed-fraction below this collapses a build as code-identical. Default 0 = only exact
/// code-identity collapses (any real change keeps the build code-distinct).
#[arg(long, default_value_t = 0.0)]
skip_below: f64,
/// changed-fraction at or above this marks a toolchain shift = an era boundary (default 0.20; see
/// `classify-change --shift-above` for what that number was measured against).
#[arg(long, default_value_t = 0.20)]
shift_above: f64,
#[arg(long)]
threads: Option<usize>,
/// Write the selection manifest here (JSON); prints to stdout if omitted.
#[arg(long)]
out: Option<PathBuf>,
},
}
/// A command's `--lib`, defaulting to the game's server library when unset.
fn lib_or_default(prof: &profile::GameProfile, lib: Option<String>) -> String {
lib.unwrap_or_else(|| prof.server_lib.to_string())
}
/// Resolve the catalogue for the model commands (`corpus-model`/`fold-model`) from either a `--seed` bundle
/// (release form) or a loose `--catalogue` file — never both; `catalogue` declares the conflict, so the
/// `None` arm here means the flag was genuinely absent. The seed's catalogue section parses to the same
/// functions as the loose `needed-functions.json`, so the distilled/folded model is identical either way.
/// When a seed is given, its sections unpack under a `.seed` dir beside `out` (as `produce` does beside
/// its out-dir).
fn model_catalogue(
prof: &profile::GameProfile,
seed: Option<PathBuf>,
catalogue: Option<PathBuf>,
out: &std::path::Path,
) -> Result<PathBuf> {
match seed {
Some(s) => {
let work = out
.parent()
.unwrap_or_else(|| std::path::Path::new("."))
.join(".seed");
Ok(unpack_seed(prof, &s, &work)?.catalogue)
}
None => catalogue.context("pass --seed <bundle> or --catalogue <file>"),
}
}
fn main() -> Result<()> {
let cli = Cli::parse();
// Thread the resolved profile as an explicit parameter rather than a process-wide global, so the
// engine stays reusable per call.
let profile = match cli.game {
Game::Cs2 => &profile::CS2,
Game::Dota2 => &profile::DOTA,
};
match cli.cmd {
Cmd::IntegrationTest {
game_dir,
build,
lib,
wait,
map,
bots,
gamedata,
out,
keep,
fuzz_iterations,
} => {
let map = map.unwrap_or_else(|| profile.default_map.to_string());
let lib = lib_or_default(profile, lib);
integration_test_cmd(
profile,
&game_dir,
build.as_deref(),
&lib,
wait,
&map,
bots,
gamedata.as_deref(),
out.as_deref(),
keep,
fuzz_iterations,
)
}
Cmd::Produce {
game_dir,
build,
lib,
seed,
catalogue,
corpus,
corpus_model,
target,
promotable,
candidates,
full_names,
extra_offsets,
extra_sigs,
prototypes,
semantics,
ehandle_classes,
sig_cap,
version,
out_dir,
class_scope,
wait,
map,
bots,
} => {
// derive inputs come from a single --seed bundle (release form) or the loose flags (dev/verify).
// The bundle arm reads NONE of the loose bindings, which is only honest because each of them
// declares `conflicts_with = "seed"` — clap rejects the mix before dispatch rather than letting
// this arm drop an explicitly passed input on the floor.
let inputs = match seed {
Some(s) => unpack_seed(profile, &s, &out_dir.join(".seed"))?,
None => SeedInputs {
catalogue: catalogue.context("pass --seed <bundle> or --catalogue <file>")?,
promotable,
candidates,
full_names,
extra_offsets,
extra_sigs,
},
};
let map = map.unwrap_or_else(|| profile.default_map.to_string());
let lib = lib_or_default(profile, lib);
produce_cmd(ProduceArgs {
prof: profile,
game: game_dir.as_deref(),
build: build.as_deref(),
lib: &lib,
catalogue: &inputs.catalogue,
corpus: corpus.as_deref(),
corpus_model: corpus_model.as_deref(),
class_scope,
target: &target,
promotable: inputs.promotable.as_deref(),
candidates: inputs.candidates.as_deref(),
full_names: inputs.full_names.as_deref(),
extra_offsets: inputs.extra_offsets.as_deref(),
extra_sigs: inputs.extra_sigs.as_deref(),
prototypes: prototypes.as_deref(),
semantics: semantics.as_deref(),
ehandle_classes: ehandle_classes.as_deref(),
sig_cap,
version: &version,
out_dir: &out_dir,
wait,
map: &map,
bots,
})
}
Cmd::CorpusModel {
seed,
catalogue,
corpus,
class_scope,
out,
} => {
let cat = model_catalogue(profile, seed, catalogue, &out)?;
corpus_model_cmd(profile, &cat, &corpus, class_scope, &out)
}
Cmd::FoldModel {
model,
seed,
catalogue,
build,
class_scope,
out,
} => {
let cat = model_catalogue(profile, seed, catalogue, &out)?;
fold_model_cmd(
profile,
load_model(&model)?,
&cat,
&build,
class_scope,
&out,
)
}
Cmd::Backfill {
corpus,
corpus_model,
lib,
names,
threads,
out,
} => {
let lib = lib_or_default(profile, lib);
backfill_cmd(
profile,
corpus.as_deref(),
corpus_model.as_deref(),
&lib,
&names,
threads,
out.as_deref(),
)
}
Cmd::ClassifyChange {
prev,
new,
lib,
skip_below,
shift_above,
json,
} => {
let lib = lib_or_default(profile, lib);
classify_change_cmd(&prev, &new, &lib, skip_below, shift_above, json)
}
Cmd::FilterCorpus {
corpus,
lib,
skip_below,
shift_above,
threads,
out,
} => {
let lib = lib_or_default(profile, lib);
filter_corpus_cmd(
profile,
&corpus,
&lib,
skip_below,
shift_above,
out.as_deref(),
threads,
)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use clap::CommandFactory;
/// A dropped input is a silent skip, and this is the one place the CLI could produce one: the `--seed`
/// arms unpack every loose input themselves and never read the loose bindings, so an undeclared
/// conflict means `produce --seed s.json --full-names f.json` runs to exit 0 with `--full-names`
/// ignored — and a monolith with no experimental tier is exactly what a game with no naming harvest
/// legitimately ships, so no collapse floor downstream can tell the two apart.
#[test]
fn a_seed_bundle_refuses_the_loose_inputs_rather_than_ignoring_them() {
Cli::command().debug_assert();
let parse = |argv: &[&str]| {
let full: Vec<&str> = std::iter::once("source2rosetta")
.chain(argv.iter().copied())
.collect();
Cli::try_parse_from(&full)
};
let refused = |argv: &[&str]| {
assert!(
parse(argv).is_err(),
"accepted, so one of these inputs is silently dropped: {argv:?}"
);
};
let produce = ["produce", "--target", "t", "--out-dir", "o", "--seed", "s"];
for flag in [
"--catalogue",
"--promotable",
"--candidates",
"--full-names",
"--extra-offsets",
"--extra-sigs",
] {
refused(&[&produce[..], &[flag, "x"]].concat());
}
let model = ["--out", "o", "--seed", "s", "--catalogue", "c"];
refused(&[&["corpus-model", "--corpus", "c"][..], &model].concat());
refused(&[&["fold-model", "--model", "m", "--build", "b"][..], &model].concat());
// Either form ALONE still parses — the conflict must not have made the loose form unusable.
assert!(parse(&produce).is_ok());
let loose = ["--catalogue", "c", "--full-names", "f"];
let bare = &produce[..produce.len() - 2]; // the same command minus `--seed s`
assert!(parse(&[bare, &loose[..]].concat()).is_ok());
}
}