482 lines
21 KiB
Rust
482 lines
21 KiB
Rust
//! source2rosetta — CLI front-end. A thin clap layer over `source2rosetta::pipeline`: parse args,
|
|
//! select the game profile, dispatch to the engine.
|
|
|
|
use anyhow::{Context, Result};
|
|
use clap::{Parser, Subcommand};
|
|
use source2rosetta::pipeline::{
|
|
ClassScope, backfill_cmd, corpus_model_cmd, fold_model_cmd, load_model,
|
|
};
|
|
use source2rosetta::produce::{
|
|
ProduceArgs, SeedInputs, classify_change_cmd, filter_corpus_cmd, integration_test_cmd,
|
|
produce_cmd, unpack_seed,
|
|
};
|
|
use source2rosetta::profile;
|
|
use std::path::PathBuf;
|
|
|
|
/// The game whose profile drives lib/pawn/launch/dead-weight knobs. Adding a game = a profile const + an arm.
|
|
#[derive(Clone, Copy, clap::ValueEnum)]
|
|
enum Game {
|
|
#[value(alias = "csgo")]
|
|
Cs2,
|
|
#[value(alias = "dota")]
|
|
Dota2,
|
|
}
|
|
|
|
#[derive(Parser)]
|
|
#[command(
|
|
name = "source2rosetta",
|
|
about = "Locate Source-2 engine functions across builds"
|
|
)]
|
|
struct Cli {
|
|
/// Which game's profile to use — selects lib/pawn/launch/game-key/dead-weight knobs. Source-2-generic
|
|
/// behavior is unaffected; only the game-specific paths read the selected profile. (Known: cs2, dota2.)
|
|
#[arg(long, global = true, value_enum, default_value = "cs2")]
|
|
game: Game,
|
|
#[command(subcommand)]
|
|
cmd: Cmd,
|
|
}
|
|
|
|
#[derive(Subcommand)]
|
|
enum Cmd {
|
|
/// Own the process end-to-end for CI — no human, no mod: LAUNCH a VANILLA dedicated server for the
|
|
/// selected `--game`, populate it (CS2: bots on an empty deathmatch; a pawn-less game like Dota waits
|
|
/// for its `ready_class` proxy instead), then verify the derived gamedata against it — the schema oracle,
|
|
/// a semantic ptrace CALL on a live pawn (pawn games only), and (with --gamedata) a full validate-live.
|
|
/// (Disable metamod in the game's `gameinfo.gi` for a truly vanilla run — no hooks, clean pass/fail.)
|
|
IntegrationTest {
|
|
/// Game root (contains `bin/linuxsteamrt64/<executable>` and the content dir).
|
|
#[arg(long = "game-dir")]
|
|
game_dir: PathBuf,
|
|
/// Dir holding the on-disk libserver.so for the offline reference (defaults to --game).
|
|
#[arg(long)]
|
|
build: Option<PathBuf>,
|
|
/// Server library to derive from; defaults to the active game's server lib.
|
|
#[arg(long)]
|
|
lib: Option<String>,
|
|
/// Seconds to wait for the server to come up and bots to spawn alive.
|
|
#[arg(long, default_value_t = 60)]
|
|
wait: u64,
|
|
#[arg(long)] // default resolved from the active game profile at dispatch
|
|
map: Option<String>,
|
|
/// Number of bots to fill the server with.
|
|
#[arg(long, default_value_t = 9)]
|
|
bots: u32,
|
|
/// Optional gamedata json to also validate-live against the running server.
|
|
#[arg(long)]
|
|
gamedata: Option<PathBuf>,
|
|
/// Write the validated (kept) gamedata here (with --gamedata) — so this one command owns the
|
|
/// server AND persists the live-validated result, no separate validate-live needed.
|
|
#[arg(long)]
|
|
out: Option<PathBuf>,
|
|
/// Leave the launched server running instead of killing it after the test.
|
|
#[arg(long)]
|
|
keep: bool,
|
|
/// With --gamedata, also run the LIVE fuzzer against this same server for N randomized probes
|
|
/// (0 = off). Reuses the launched server — no separate `fuzz-live` run needed for CI.
|
|
#[arg(long, default_value_t = 500)]
|
|
fuzz_iterations: usize,
|
|
},
|
|
/// The whole per-game build in ONE in-memory command: derive → fold → (if `--game-dir` is given)
|
|
/// validate-live + typed netvars → fold model, writing the release set (`gamedata-`/`netvars-`/`model-`/
|
|
/// `manifest`) into --out-dir. No per-stage intermediate files. **Pass `--game-dir` for a full,
|
|
/// live-validated build; omit it for a fast OFFLINE build (gamedata + model only, no server).**
|
|
Produce {
|
|
/// A launchable game install → the FULL build (boots a server for validate-live + typed netvars).
|
|
/// OMIT for an offline build (gamedata + model only). The offline/full switch — no separate flag.
|
|
#[arg(long = "game-dir")]
|
|
game_dir: Option<PathBuf>,
|
|
/// Dir holding the on-disk libs for make-sig + live validation (defaults to --game-dir, else --target).
|
|
#[arg(long)]
|
|
build: Option<PathBuf>,
|
|
/// Server library to derive from; defaults to the active game's server lib.
|
|
#[arg(long)]
|
|
lib: Option<String>,
|
|
/// One bundled seed (catalogue + naming sections) — the release form. Replaces the loose
|
|
/// --catalogue/--promotable/--candidates/--full-names/--extra-offsets/--extra-sigs flags.
|
|
#[arg(long)]
|
|
seed: Option<PathBuf>,
|
|
/// Function catalogue (loose form; omit when using --seed).
|
|
#[arg(long)]
|
|
catalogue: Option<PathBuf>,
|
|
/// Corpus-signal source A: the raw build binaries to fingerprint on the fly. Exactly ONE of
|
|
/// --corpus / --corpus-model is required (--corpus-model is the production forward-derive path).
|
|
#[arg(long)]
|
|
corpus: Option<PathBuf>,
|
|
/// Corpus-signal source B: a distilled `model-<game>.json` — forward-derives from the model + only the
|
|
/// target binary (no corpus). Also triggers the sidecar fold (model N → N+1). See --corpus.
|
|
#[arg(long)]
|
|
corpus_model: Option<PathBuf>,
|
|
/// The build DIRECTORY to DERIVE gamedata from — the primary input (its libs are searched by name).
|
|
/// A bare `.so` path is not searched; pass the directory that contains it. REQUIRED.
|
|
#[arg(long)]
|
|
target: PathBuf,
|
|
/// Optional: names eligible for promotion into high_confidence (from the naming producer flow).
|
|
/// Omit to promote nothing — the catalogue still derives in full.
|
|
#[arg(long)]
|
|
promotable: Option<PathBuf>,
|
|
/// Optional: prefiltered per-address context for those names (`{"candidates": [...]}`). Omit for none.
|
|
#[arg(long)]
|
|
candidates: Option<PathBuf>,
|
|
/// Optional: the full-slice name universe. When set, the monolith also carries an `experimental`
|
|
/// tier — the least-filtered inclusion band (every name guess, graded, each with a resolvable
|
|
/// locator but an UNVERIFIED name).
|
|
#[arg(long)]
|
|
full_names: Option<PathBuf>,
|
|
/// Multilib ground-truth vtable offsets to fold as high_confidence — `{lib: [{name,class,slot}]}`
|
|
/// (e.g. the macOS symbol transfer). Folded directly, bypassing the candidate gate.
|
|
#[arg(long)]
|
|
extra_offsets: Option<PathBuf>,
|
|
/// Multilib non-virtual names to fold as sigs — `{lib: [{name,addr}]}`; `make_sig` runs per lib.
|
|
#[arg(long)]
|
|
extra_sigs: Option<PathBuf>,
|
|
/// Byte budget for signatures the FOLD generates (the extrapolated tiers). The derive's own
|
|
/// `core` sigs use a separate fixed budget — this flag does not widen those.
|
|
#[arg(long, default_value_t = 400)]
|
|
sig_cap: usize,
|
|
#[arg(long, default_value = "vX")]
|
|
version: String,
|
|
#[arg(long)]
|
|
out_dir: PathBuf,
|
|
/// Class scope for the sidecar model fold — must match the scope the input model was distilled with.
|
|
#[arg(long, value_enum, default_value = "clean")]
|
|
class_scope: ClassScope,
|
|
#[arg(long, default_value_t = 90)]
|
|
wait: u64,
|
|
#[arg(long)] // default resolved from the active game profile at dispatch
|
|
map: Option<String>,
|
|
#[arg(long, default_value_t = 9)]
|
|
bots: u32,
|
|
},
|
|
/// Distill the whole corpus into a shippable model (vtable-alignment hops + reference fingerprints
|
|
/// + slot timelines) so derivation needs only the model + the target binary, not the 86 GB corpus.
|
|
CorpusModel {
|
|
/// One bundled seed — the release form; its catalogue section is what gets distilled. Replaces the
|
|
/// loose --catalogue (naming sections are ignored here — the model tracks catalogue names only).
|
|
#[arg(long)]
|
|
seed: Option<PathBuf>,
|
|
/// Function catalogue (loose form; omit when using --seed).
|
|
#[arg(long)]
|
|
catalogue: Option<PathBuf>,
|
|
#[arg(long)]
|
|
corpus: PathBuf,
|
|
/// Which classes get vtable-slot hops: `clean` (every real game class — the default; enough for any
|
|
/// modding offset to derive model-only), `all` (also template/protobuf/NetworkVar junk), or
|
|
/// `catalogue` (only what the catalogue names). CI compresses the model, so on-disk size isn't shipped.
|
|
#[arg(long, value_enum, default_value = "clean")]
|
|
class_scope: ClassScope,
|
|
#[arg(long)]
|
|
out: PathBuf,
|
|
},
|
|
/// Incrementally fold ONE new build into an existing model: `model N + build → model N+1`, equal to a
|
|
/// full re-distill over the same builds but reading only the model + the one binary (no corpus). The
|
|
/// production update path — keeps the model fresh per build without re-reading history.
|
|
FoldModel {
|
|
/// The existing model N (carries the `abi_obs` window the fold re-windows).
|
|
#[arg(long)]
|
|
model: PathBuf,
|
|
/// One bundled seed — the release form; its catalogue section is folded. Replaces the loose --catalogue.
|
|
#[arg(long)]
|
|
seed: Option<PathBuf>,
|
|
/// Function catalogue (loose form; omit when using --seed). Must match the model's distill catalogue.
|
|
#[arg(long)]
|
|
catalogue: Option<PathBuf>,
|
|
/// The one new build dir to fold in (holds the just-updated libserver.so etc.).
|
|
#[arg(long)]
|
|
build: PathBuf,
|
|
/// Must match the scope the model was distilled with (`clean` default).
|
|
#[arg(long, value_enum, default_value = "clean")]
|
|
class_scope: ClassScope,
|
|
#[arg(long)]
|
|
out: PathBuf,
|
|
},
|
|
/// Back-fill cross-build history for extrapolated (T3) names: for each {name, anchor} pair, resolve
|
|
/// the anchor STRING uniquely in every corpus build (the same string-anchor locator as `anchor`), so
|
|
/// a name that was a single-build guess gains a real timeline. Reports per-name history depth +
|
|
/// consistency-since-first-appearance — the measure of how many T3 names graduate to first-class
|
|
/// (a function anchored across hundreds of builds is high-confidence regardless of its T3 origin).
|
|
Backfill {
|
|
/// Raw build binaries — needed for the string-anchor half (locating a sig/self-named function in
|
|
/// each historical build). Omit to run only the model-only offset half.
|
|
#[arg(long)]
|
|
corpus: Option<PathBuf>,
|
|
/// Distilled corpus model — its `hops` back-fill an OFFSET function's vtable-slot timeline with
|
|
/// NO binaries (the community-PR-of-a-vtable-method path). Omit to run only the string half.
|
|
#[arg(long)]
|
|
corpus_model: Option<PathBuf>,
|
|
/// Server library to derive from; defaults to the active game's server lib.
|
|
#[arg(long)]
|
|
lib: Option<String>,
|
|
/// JSON array of {name, tier?, anchor?, class?, slot?}: `anchor` (a distinctive string it
|
|
/// references — its own name for self-named) drives the string half; `class`+`slot` drive the
|
|
/// model-hops half.
|
|
#[arg(long)]
|
|
names: PathBuf,
|
|
#[arg(long)]
|
|
threads: Option<usize>,
|
|
/// Write the per-name timeline report here.
|
|
#[arg(long)]
|
|
out: Option<PathBuf>,
|
|
},
|
|
/// Classify how much a library changed between two builds — the CI branch primitive. Enumerates every
|
|
/// function (`.eh_frame`) in each build and compares their bodies with the position-dependent bytes
|
|
/// (RIP-relative displacements + near-branch targets) masked out, so the verdict is shift-invariant:
|
|
/// a pure layout move (bodies unchanged, addresses shifted) reads as UNCHANGED, unlike a raw byte diff.
|
|
/// Prints `skip` (nothing meaningful changed → no release), `normal` (an ordinary patch → re-derive) or
|
|
/// `shift` (a toolchain/compiler change moved ~every function's codegen at once → re-derive, and the
|
|
/// derive leans harder on the string-anchor/vtable recovery paths) plus the exact % of the new build's
|
|
/// functions whose body isn't byte-identical to the
|
|
/// previous build's. The thresholds are heuristic defaults — calibrate `--skip-below`/`--shift-above`
|
|
/// against real adjacent-vs-toolchain-jump pairs.
|
|
ClassifyChange {
|
|
/// Previous build: a `.so` file directly, or a build dir to find `--lib` under.
|
|
#[arg(long)]
|
|
prev: PathBuf,
|
|
/// New build: a `.so` file directly, or a build dir to find `--lib` under.
|
|
#[arg(long)]
|
|
new: PathBuf,
|
|
/// Server library to derive from; defaults to the active game's server lib.
|
|
#[arg(long)]
|
|
lib: Option<String>,
|
|
/// Extra `skip` tolerance: a changed-fraction below this also counts as `skip`. Default 0 —
|
|
/// only a code-IDENTICAL build (0 functions changed) skips, so any real patch re-derives. Raise
|
|
/// it (e.g. 0.01) to also skip changes under N%. (Calibration on 339 CS2 pairs: 311 are
|
|
/// code-identical, real patches touch <=6 functions / <=0.08%, the 2 toolchain jumps are 34%/53%.)
|
|
#[arg(long, default_value_t = 0.0)]
|
|
skip_below: f64,
|
|
/// changed-fraction at or above this = `shift`. Default 0.20 — the CS2 corpus's real patches top
|
|
/// out near 0.08% while its two toolchain jumps are 34%/53%, so 20% cleanly separates them with
|
|
/// wide margin and (unlike 40%) doesn't misclassify the 34% jump as an ordinary patch.
|
|
#[arg(long, default_value_t = 0.20)]
|
|
shift_above: f64,
|
|
/// Emit a machine-readable JSON object instead of the human summary.
|
|
#[arg(long)]
|
|
json: bool,
|
|
},
|
|
/// Stage-1 change-aware corpus filter: walk a game's builds chronologically, collapse runs of
|
|
/// code-identical builds (bodies unchanged, only relocations moved) to ONE representative, label each
|
|
/// surviving transition normal/shift, and segment the timeline into toolchain ERAS (cut at shifts).
|
|
/// Writes a selection manifest (code-distinct kept builds + era/drift each). Lossless for the per-game
|
|
/// facts; the distinct-build set `corpus-model` distills. Digests each build once.
|
|
FilterCorpus {
|
|
#[arg(long)]
|
|
corpus: PathBuf,
|
|
/// Server library to derive from; defaults to the active game's server lib.
|
|
#[arg(long)]
|
|
lib: Option<String>,
|
|
/// changed-fraction below this collapses a build as code-identical. Default 0 = only exact
|
|
/// code-identity collapses (any real change keeps the build code-distinct).
|
|
#[arg(long, default_value_t = 0.0)]
|
|
skip_below: f64,
|
|
/// changed-fraction at or above this marks a toolchain shift = an era boundary (default 0.20).
|
|
#[arg(long, default_value_t = 0.20)]
|
|
shift_above: f64,
|
|
#[arg(long)]
|
|
threads: Option<usize>,
|
|
/// Write the selection manifest here (JSON); prints to stdout if omitted.
|
|
#[arg(long)]
|
|
out: Option<PathBuf>,
|
|
},
|
|
}
|
|
|
|
/// A command's `--lib`, defaulting to the game's server library when unset.
|
|
fn lib_or_default(prof: &profile::GameProfile, lib: Option<String>) -> String {
|
|
lib.unwrap_or_else(|| prof.server_lib.to_string())
|
|
}
|
|
|
|
/// Resolve the catalogue for the model commands (`corpus-model`/`fold-model`) from either a `--seed` bundle
|
|
/// (release form) or a loose `--catalogue` file. The seed's catalogue section parses to the same functions as
|
|
/// the loose `needed-functions.json`, so the distilled/folded model is identical either way. When a seed is
|
|
/// given, its sections unpack under a `.seed` dir beside `out` (as `produce` does beside its out-dir).
|
|
fn model_catalogue(
|
|
prof: &profile::GameProfile,
|
|
seed: Option<PathBuf>,
|
|
catalogue: Option<PathBuf>,
|
|
out: &std::path::Path,
|
|
) -> Result<PathBuf> {
|
|
match seed {
|
|
Some(s) => {
|
|
let work = out
|
|
.parent()
|
|
.unwrap_or_else(|| std::path::Path::new("."))
|
|
.join(".seed");
|
|
Ok(unpack_seed(prof, &s, &work)?.catalogue)
|
|
}
|
|
None => catalogue.context("pass --seed <bundle> or --catalogue <file>"),
|
|
}
|
|
}
|
|
|
|
fn main() -> Result<()> {
|
|
let cli = Cli::parse();
|
|
// Thread the resolved profile as an explicit parameter rather than a process-wide global, so the
|
|
// engine stays reusable per call.
|
|
let profile = match cli.game {
|
|
Game::Cs2 => &profile::CS2,
|
|
Game::Dota2 => &profile::DOTA,
|
|
};
|
|
match cli.cmd {
|
|
Cmd::IntegrationTest {
|
|
game_dir,
|
|
build,
|
|
lib,
|
|
wait,
|
|
map,
|
|
bots,
|
|
gamedata,
|
|
out,
|
|
keep,
|
|
fuzz_iterations,
|
|
} => {
|
|
let map = map.unwrap_or_else(|| profile.default_map.to_string());
|
|
let lib = lib_or_default(profile, lib);
|
|
integration_test_cmd(
|
|
profile,
|
|
&game_dir,
|
|
build.as_deref(),
|
|
&lib,
|
|
wait,
|
|
&map,
|
|
bots,
|
|
gamedata.as_deref(),
|
|
out.as_deref(),
|
|
keep,
|
|
fuzz_iterations,
|
|
)
|
|
}
|
|
Cmd::Produce {
|
|
game_dir,
|
|
build,
|
|
lib,
|
|
seed,
|
|
catalogue,
|
|
corpus,
|
|
corpus_model,
|
|
target,
|
|
promotable,
|
|
candidates,
|
|
full_names,
|
|
extra_offsets,
|
|
extra_sigs,
|
|
sig_cap,
|
|
version,
|
|
out_dir,
|
|
class_scope,
|
|
wait,
|
|
map,
|
|
bots,
|
|
} => {
|
|
// derive inputs come from a single --seed bundle (release form) or the loose flags (dev/verify).
|
|
let inputs = match seed {
|
|
Some(s) => unpack_seed(profile, &s, &out_dir.join(".seed"))?,
|
|
None => SeedInputs {
|
|
catalogue: catalogue.context("pass --seed <bundle> or --catalogue <file>")?,
|
|
promotable,
|
|
candidates,
|
|
full_names,
|
|
extra_offsets,
|
|
extra_sigs,
|
|
},
|
|
};
|
|
let map = map.unwrap_or_else(|| profile.default_map.to_string());
|
|
let lib = lib_or_default(profile, lib);
|
|
produce_cmd(ProduceArgs {
|
|
prof: profile,
|
|
game: game_dir.as_deref(),
|
|
build: build.as_deref(),
|
|
lib: &lib,
|
|
catalogue: &inputs.catalogue,
|
|
corpus: corpus.as_deref(),
|
|
corpus_model: corpus_model.as_deref(),
|
|
class_scope,
|
|
target: &target,
|
|
promotable: inputs.promotable.as_deref(),
|
|
candidates: inputs.candidates.as_deref(),
|
|
full_names: inputs.full_names.as_deref(),
|
|
extra_offsets: inputs.extra_offsets.as_deref(),
|
|
extra_sigs: inputs.extra_sigs.as_deref(),
|
|
sig_cap,
|
|
version: &version,
|
|
out_dir: &out_dir,
|
|
wait,
|
|
map: &map,
|
|
bots,
|
|
})
|
|
}
|
|
Cmd::CorpusModel {
|
|
seed,
|
|
catalogue,
|
|
corpus,
|
|
class_scope,
|
|
out,
|
|
} => {
|
|
let cat = model_catalogue(profile, seed, catalogue, &out)?;
|
|
corpus_model_cmd(profile, &cat, &corpus, class_scope, &out)
|
|
}
|
|
Cmd::FoldModel {
|
|
model,
|
|
seed,
|
|
catalogue,
|
|
build,
|
|
class_scope,
|
|
out,
|
|
} => {
|
|
let cat = model_catalogue(profile, seed, catalogue, &out)?;
|
|
fold_model_cmd(
|
|
profile,
|
|
load_model(&model)?,
|
|
&cat,
|
|
&build,
|
|
class_scope,
|
|
&out,
|
|
)
|
|
}
|
|
Cmd::Backfill {
|
|
corpus,
|
|
corpus_model,
|
|
lib,
|
|
names,
|
|
threads,
|
|
out,
|
|
} => {
|
|
let lib = lib_or_default(profile, lib);
|
|
backfill_cmd(
|
|
profile,
|
|
corpus.as_deref(),
|
|
corpus_model.as_deref(),
|
|
&lib,
|
|
&names,
|
|
threads,
|
|
out.as_deref(),
|
|
)
|
|
}
|
|
Cmd::ClassifyChange {
|
|
prev,
|
|
new,
|
|
lib,
|
|
skip_below,
|
|
shift_above,
|
|
json,
|
|
} => {
|
|
let lib = lib_or_default(profile, lib);
|
|
classify_change_cmd(&prev, &new, &lib, skip_below, shift_above, json)
|
|
}
|
|
Cmd::FilterCorpus {
|
|
corpus,
|
|
lib,
|
|
skip_below,
|
|
shift_above,
|
|
threads,
|
|
out,
|
|
} => {
|
|
let lib = lib_or_default(profile, lib);
|
|
filter_corpus_cmd(
|
|
profile,
|
|
&corpus,
|
|
&lib,
|
|
skip_below,
|
|
shift_above,
|
|
out.as_deref(),
|
|
threads,
|
|
)
|
|
}
|
|
}
|
|
}
|