//! CI orchestration + the LIVE half of the engine. `produce` runs the whole per-game build in one //! long-running command (derive → fold → validate-live → typed netvars → fold-model), merging every //! stage's account of a function into the one shipped artifact; `classify-change` and `filter-corpus` are the CI *branch* primitives (is this buildid worth //! a release? which corpus builds are code-distinct?). Everything that attaches to and drives a RUNNING //! server lives here, not in `pipeline`: the semantic oracle (`run_live_oracle`, pawn probing, `fuzz_live_run`), //! `integration_test_cmd`, the live validators (`validate_live_cmd`/`verify_live_cmd` + their sig/offset //! checks), and `launch_bots_server`. `pipeline` stays the pure OFFLINE derivation engine these consume. use crate::elf::CodeImage; use crate::locate::{find_file, load_lib}; use crate::par::{default_threads, parallel_map}; use crate::pipeline::{ ClassScope, CorpusModel, CorpusSource, FoldArgs, Folded, GdMap, annotate_validation, build_date, build_gamedata_cmd, find_builds, fold_model_cmd, gamedata, label_of, lib_filename, load_model, read_gamedata_str, }; use crate::profile::{self, GameProfile}; use crate::sig::Pattern; use crate::taxonomy::{class_of, is_query_method}; use crate::{abi, emit, live, model, render, rtti, schema}; use anyhow::{Context, Result, bail, ensure}; use serde_json::{Value, json}; use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Path, PathBuf}; /// The per-game derive INPUTS a `--seed` bundle carries, whether passed as loose files or unpacked from /// the bundle: the catalogue plus the naming-harvest sections, and a folded snapshot of the contributions /// inbox. /// /// NOT the complete input surface, and the difference matters when tracking down where a fact came from. /// The judged/authored inputs stay separate flags (`--prototypes`, `--semantics`, `--ehandle-classes`), /// the corpus model is its own streamed-write artifact (`--corpus-model`), and on the loose-flag path the /// derive additionally reads `/contributions//*.json` implicitly — which is /// why `unpack_seed` writes the bundle's contributions beside the catalogue rather than anywhere else. pub struct SeedInputs { pub catalogue: PathBuf, /// Optional bring-up refinements — a seed without them still derives the full catalogue. pub promotable: Option, pub candidates: Option, pub full_names: Option, pub extra_offsets: Option, pub extra_sigs: Option, } /// Un-bundle a `--seed` file — one JSON object whose sections are the verbatim contents of the former /// loose files — into on-disk inputs under `work/`, so the derive reads them exactly as before (the fold /// re-parses, so a serde round-trip of a section changes nothing downstream). Only `catalogue` is required; /// every other section is optional and a missing one stays `None` — a game with no naming harvest yet ships /// a seed with no `promotable`, and `seed-dota2.json` carries no `contributions`. pub fn unpack_seed(prof: &GameProfile, seed: &Path, work: &Path) -> Result { let v: Value = serde_json::from_str(&std::fs::read_to_string(seed)?) .with_context(|| format!("parse seed {}", seed.display()))?; std::fs::create_dir_all(work)?; let dump = |key: &str, fname: &str| -> Result> { match v.get(key) { Some(section) if !section.is_null() => { let p = work.join(fname); std::fs::write(&p, serde_json::to_string(section)?)?; Ok(Some(p)) } _ => Ok(None), } }; // Contributions unpack BESIDE the catalogue (work/contributions//), so the derive's // `load_contributions` (which reads catalogue.parent()) folds them exactly as the loose form does from // mappings/contributions/. The repo dir stays the human PR inbox; the seed carries a folded snapshot. let cdir = work.join("contributions").join(prof.game_key); // CLEARED first, and it has to be: `load_contributions` folds every file it finds in this directory, // so a leftover from an earlier run under a reused --out-dir would be merged into a build whose seed // never mentioned it. The artifact would then depend on what happened to be on disk rather than on // its inputs, which is the one property a reproducible derive cannot give up. if cdir.exists() { std::fs::remove_dir_all(&cdir) .with_context(|| format!("clear stale contributions in {}", cdir.display()))?; } if let Some(Value::Object(files)) = v.get("contributions") { std::fs::create_dir_all(&cdir)?; for (fname, content) in files { std::fs::write(cdir.join(fname), serde_json::to_string(content)?)?; } } let need = |o: Option, key: &str| { o.with_context(|| format!("seed missing required '{key}' section")) }; Ok(SeedInputs { catalogue: need(dump("catalogue", "needed-functions.json")?, "catalogue")?, promotable: dump("promotable", "promotable.json")?, candidates: dump("candidates", "candidates.json")?, full_names: dump("full_names", "full-names.json")?, extra_offsets: dump("extra_offsets", "extra-offsets.json")?, extra_sigs: dump("extra_sigs", "extra-sigs.json")?, }) } /// Read the authored function descriptions. A repo input, so a parse failure is a hard error rather /// than a silently empty join — an unreadable file and a file describing nothing are the same size in /// the output otherwise. fn read_semantics(path: &Path) -> Result> { #[derive(serde::Deserialize)] struct SemanticsDoc { descriptions: BTreeMap, } let text = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; let doc: SemanticsDoc = serde_json::from_str(&text).with_context(|| format!("parse {}", path.display()))?; Ok(doc.descriptions) } /// Inputs to [`produce_cmd`], grouped so the CLI passes ONE named-field value instead of a long positional /// list (where two same-typed `Option<&Path>` could silently transpose). Built by the clap front-end after /// it resolves the corpus source and the seed-or-loose derive inputs. pub struct ProduceArgs<'a> { pub prof: &'a GameProfile, /// A launchable game install. `Some` → the full build (validate-live + typed netvars); `None` → OFFLINE /// (no server, so no live validation and a `null` schema). The whole offline/full switch — there /// is no separate flag. pub game: Option<&'a Path>, pub build: Option<&'a Path>, pub lib: &'a str, pub catalogue: &'a Path, /// Reference-signal source paths — exactly one is `Some`, enforced at the top of `produce_cmd` rather /// than only by clap, because an embedder calling `produce_cmd` directly bypasses the CLI. A /// `--corpus-model` is parsed ONCE here so the derive can borrow it and the sidecar fold can consume /// the same instance. pub corpus: Option<&'a Path>, pub corpus_model: Option<&'a Path>, /// Class scope for the sidecar model fold — must match the scope the input model was distilled with. pub class_scope: ClassScope, pub target: &'a Path, pub promotable: Option<&'a Path>, pub candidates: Option<&'a Path>, pub full_names: Option<&'a Path>, pub extra_offsets: Option<&'a Path>, pub extra_sigs: Option<&'a Path>, /// Declared prototypes to JUDGE against this build's measured footprints. Static repo input, not /// rolling state, so it is passed as a path rather than fetched. pub prototypes: Option<&'a Path>, /// Authored function descriptions (`mappings/semantics-.json`), folded in beside each /// function. A repo input on the same terms as `prototypes` — reviewed, committed, and keyed on the /// NAME, so it survives every build that does not rename a function. Optional. pub semantics: Option<&'a Path>, /// Valve's `PVAL_EHANDLE` entity-class naming (`mappings/ehandle-classes.json`) — a static repo /// input the Pulse surface is enriched with. Optional. pub ehandle_classes: Option<&'a Path>, pub sig_cap: usize, pub version: &'a str, pub out_dir: &'a Path, pub wait: u64, pub map: &'a str, pub bots: u32, } /// The whole per-game build in ONE command, entirely in memory — derive → fold → (if a game is given) /// validate-live + typed netvars → judge prototypes → merge → fold model — writing the release set ONCE /// (`rosetta-.json` + `model-.json` [corpus-model source] + `manifest.json`). No per-stage /// intermediate files: the derive hands its rendered gamedata straight to the fold, the fold's catalogue /// is annotated live in place, the schema is read straight off the process, and the merge folds all of /// it together. Offline vs full is decided solely by whether a launchable `game` is present. pub fn produce_cmd(a: ProduceArgs) -> Result<()> { let ProduceArgs { prof, game, build, lib, catalogue, corpus, corpus_model, class_scope, target, promotable, candidates, full_names, extra_offsets, extra_sigs, prototypes, semantics, ehandle_classes, sig_cap, version, out_dir, wait, map, bots, } = a; // Exactly one reference-signal source, checked HERE rather than only in clap: an embedder calls // `produce_cmd` directly, and the `(Some(m), _)` arm below silently drops `--corpus` when both are // given — a derive that quietly used a different signal source than the one the caller named. ensure!( corpus.is_some() != corpus_model.is_some(), "pass exactly ONE of --corpus / --corpus-model — they are two different \ reference-signal sources and only one can be used" ); // `build` (the on-disk libs for make-sig + live validation) defaults to the game when one is given (so // the validated libs match the running server), else the derive target. let build = build.unwrap_or_else(|| game.unwrap_or(target)); std::fs::create_dir_all(out_dir)?; let p = |name: &str| out_dir.join(name); let token = prof.token; // Parse the corpus model ONCE (~571 MB for Dota): the derive borrows it below, and the sidecar fold in // step 4 consumes the same instance — no second parse. A `--corpus` (genesis) run has no model (its model // is distilled by `corpus-model`); a `--corpus-model` run rolls that model N to N+1 in the fold. let cmodel: Option = match corpus_model { Some(m) => Some(load_model(m)?), None => None, }; // The `ensure!` above has already ruled out both-or-neither. let source = match (&cmodel, corpus) { (Some(m), _) => CorpusSource::Model(m), (None, Some(c)) => CorpusSource::Binaries(c), (None, None) => bail!("pass --corpus or --corpus-model "), }; // 1. derive + fold (offline, in memory) -> the monolith + its CS# render (the string live validate checks) eprintln!("\n===== derive + fold (offline) ====="); let derived = gamedata(prof, catalogue, source, target)?; let Folded { mut mono, cssharp, mut bindings, } = build_gamedata_cmd( prof, FoldArgs { build, lib, promotable, candidates, core: &derived.core, flagged: &derived.flagged, unverified: &derived.unverified, abi: &derived.abi, anchors: &derived.anchors, sig_cap, version, full_names, extra_offsets, extra_sigs, ehandle_classes, source_build: &label_of(target), }, )?; // The DERIVED-SURFACE collapse floor, checked here rather than left to the live oracle — and it has to // be, because that gate is a pass RATE over entries that reached the gamedata document. A signature // that failed to resolve never enters it, so a derive emitting forty functions instead of four // thousand passes at 100%. Every in-binary table already has one of these; the tool's primary product // did not. See `GameProfile::min_core_functions`. let shipped = mono.meta.counts.core + mono.meta.counts.high_confidence; ensure!( shipped >= prof.min_core_functions, "derived only {shipped} shipped functions ({} core + {} high-confidence, floor {}) — a corpus \ model from another game or branch, a --target from the wrong build, or a library missing from \ the tree all land here; refusing to publish a collapsed derive", mono.meta.counts.core, mono.meta.counts.high_confidence, prof.min_core_functions ); // 2. live stages — ONLY when a launchable game is given: validate (annotate the monolith in place) + the // typed netvars (read straight off the live process). produce launches + tears down its own server. let mut netvars: Option = None; match game { Some(game) => { eprintln!("\n===== live: validate + typed netvars ====="); let mut server = launch_bots_server(prof, game, build, lib, wait, map, bots)?; let pid = server.pid; let result = (|| -> Result<()> { if let Some((_, verdicts)) = run_live_oracle(prof, pid, build, lib, Some(&cssharp), 500)? { annotate_validation(&mut mono, &verdicts); } // The one VScript field the fold cannot derive. Done here rather than inside the oracle // because it ENRICHES the artifact instead of verifying a claim about it, and because // `integration-test` — which shares the oracle — builds no artifact to enrich. if !bindings.vscript.is_empty() { let img = load_lib(build, lib)?; let live = live::LiveProcess::attach(pid)?; // `lib` is already the mapped filename (`libserver.so`), so it is passed through // rather than rebuilt. An earlier revision spelled it `lib{lib}.so`, produced // `liblibserver.so.so`, and the `if let Some` swallowed the miss — a silent zero that // looks exactly like "this build has no classes to attribute". Hence the else. match live.base(lib) { Some(base) => { let (set, classes) = attribute_vscript_classes(&live, base, &img, &mut bindings); eprintln!( " VScript classes (live-only): {set} of {} bindings attributed across \ {classes} classes", bindings.vscript.len() ); } None => eprintln!( " WARNING: {lib} is not mapped in the live server, so no VScript class \ could be attributed — the artifact will ship without the field the \ `moddota` format groups by." ), } // Live-only, so it is floored HERE rather than beside the other binding floors below: // those run on every build, and zero is the correct offline answer. See // GameProfile::min_vscript_classed. ensure!( bindings.meta.vscript_classed >= prof.min_vscript_classed, "attributed only {} VScript bindings to an owning class (floor {}) — the live \ descriptor walk likely broke; refusing to ship a binding registry the `moddota` \ format would render as empty", bindings.meta.vscript_classed, prof.min_vscript_classed ); } let nv = schema::live_schema(prof, pid, build, &label_of(target))?; // Gate the type-resolution the schema-layout oracle can't see (see NETVARS_MIN_TYPED): a // wholesale CSchemaType reshape resolves every field 'untyped' and would otherwise ship a // typeless netvars file at exit 0. Big denominator (thousands of fields), so ORACLE_MIN_SAMPLE // is always met; this is purely the all-untyped tripwire. let fields = nv.meta.typed + nv.meta.untyped; let typed_frac = if fields == 0 { 1.0 } else { nv.meta.typed as f64 / fields as f64 }; ensure!( fields < ORACLE_MIN_SAMPLE as usize || typed_frac >= NETVARS_MIN_TYPED, "typed netvars resolved only {}/{} fields ({:.1}%, floor {:.0}%) — the runtime \ CSchemaType layout likely moved; refusing to ship a typeless schema", nv.meta.typed, fields, typed_frac * 100.0, NETVARS_MIN_TYPED * 100.0 ); // The CLASS table first, because everything below rests on it — and because it is the // one table whose collapse no other check here would catch. See // GameProfile::min_schema_classes for why the live oracle's own class gate does not. ensure!( nv.classes.len() >= prof.min_schema_classes, "recovered only {} schema classes (floor {}) — the SchemaClassInfoData_t layout \ likely moved; refusing to ship a schema whose class table collapsed", nv.classes.len(), prof.min_schema_classes ); // The enum table is read by shape like the class table, so a Valve reshape yields zero // enums rather than wrong ones — safe, but silent. See GameProfile::min_schema_enums. ensure!( nv.meta.enums >= prof.min_schema_enums, "recovered only {} schema enums (floor {}) — the SchemaSystem enum-binding layout \ likely moved; refusing to ship a schema with its enum vocabulary missing", nv.meta.enums, prof.min_schema_enums ); eprintln!( " typed netvars: {} classes, {} typed fields, {} untyped", nv.classes.len(), nv.meta.typed, nv.meta.untyped ); netvars = Some(nv); Ok(()) })(); let _ = server.child.kill(); let _ = server.child.wait(); result?; // tear the server down first, THEN surface any stage error } None => { eprintln!("\n===== offline: no --game-dir -> no live validate, no typed schema =====") } } // 3. the declared callable surface. Gated PER TABLE, not on the sum: each is matched by its own record // shape, so Valve reshaping one collapses that one alone — and a summed floor stays satisfied by the // tables that still work. See GameProfile::min_pulse_bindings. for (what, got, floor) in [ ( "Pulse bindings", bindings.meta.pulse, prof.min_pulse_bindings, ), ( "typed Pulse signatures", bindings.meta.pulse_typed, prof.min_pulse_typed, ), ( "host-callable Pulse shims", bindings.meta.pulse_callable, prof.min_pulse_callable, ), ( "entity-IO records", bindings.meta.entity_inputs + bindings.meta.entity_outputs, prof.min_entity_io, ), ( "entity classnames", bindings.meta.entity_classes, prof.min_entity_classes, ), ( "console commands", bindings.meta.commands, prof.min_commands, ), ("ConVars", bindings.meta.convars, prof.min_convars), ("VScript bindings", bindings.meta.vscript, prof.min_vscript), ] { ensure!( got >= floor, "read only {got} {what} from Valve's in-binary tables (floor {floor}) — that table's layout \ likely moved; refusing to ship a release whose declared surface silently collapsed" ); } // 4. the prototype manifest — the declared parameter types, each judged against the footprint measured // in THIS build. Built before the merge rather than written beside it: a declaration and the // measurement that judges it are two facts about one function, and the merged record holds both. let manifest_abi = match prototypes { Some(pp) => { // The script VM's declared return types, keyed by the C++ name the fold used for the locator, // so a binding's `void` reaches the manifest instead of the register class measurement infers. let vs_ret: BTreeMap = bindings .vscript .iter() .filter_map(|v| v.ret.as_ref().map(|r| (v.cpp.clone(), r.clone()))) .collect(); let man = crate::prototypes::build_manifest( pp, &mono, netvars.as_ref().map(|n| &n.types), Some(&vs_ret), )?; let n = |k: &str| man.meta.counts.get(k).copied().unwrap_or(0); eprintln!( " prototypes: {} judged ({} verified, {} mismatch, {} unverified, {} return-only, \ {} ambiguous)", man.functions.len(), n("status:verified"), n("status:mismatch"), n("status:unverified"), n("status:return-only"), n("core:overloaded") + n("high_confidence:overloaded") ); Some(man) } None => None, }; // 5. authored descriptions — a repo input like the prototypes, keyed on the NAME. A name this build // does not ship simply finds no record; the join count below is what makes that visible. let descriptions = match semantics { Some(sp) => read_semantics(sp)?, None => BTreeMap::new(), }; // 6. merge and write the release artifact ONCE. Everything above is a different STAGE's account of // the same functions, so it ships as one record per function rather than four files to join. let rosetta = model::merge(mono, manifest_abi, Some(bindings), netvars, descriptions); let art_name = format!("rosetta-{token}.json"); std::fs::write(p(&art_name), serde_json::to_string_pretty(&rosetta)?) .with_context(|| format!("write {art_name}"))?; let j = &rosetta.meta.joined; eprintln!( "\n {art_name}: {} functions ({} prototypes, {} entity-IO + {} command + {} VScript bindings, \ {} descriptions), {} unresolved, schema {}", rosetta.functions.len(), j.prototypes, j.bindings.entity_input, j.bindings.command, j.bindings.vscript, j.descriptions, rosetta.unresolved.len(), match &rosetta.schema { Some(s) => format!("{} classes", s.classes.len()), None => "null (offline build)".to_string(), } ); let u = &rosetta.surfaces.unjoined; eprintln!( " surfaces: {} Pulse bindings, {} entity outputs, {} classnames, {} ConVars, \ {} declared rows with no function record of their own ({} entity-IO, {} commands, {} VScript)", rosetta.surfaces.pulse.len(), rosetta.surfaces.entity_outputs.len(), rosetta.surfaces.entity_classes.len(), rosetta.surfaces.convars.len(), u.entity_inputs.len() + u.commands.len() + u.vscript.len(), u.entity_inputs.len(), u.commands.len(), u.vscript.len() ); let mut artifacts = vec![art_name]; // 7. sidecar: fold model N -> N+1 (offline), emitted when a --corpus-model was the source. The derive has // returned, so its read-only borrow of the model is done — the fold consumes the same instance by value. if let Some(model) = cmodel { eprintln!("\n===== fold model N -> N+1 (sidecar) ====="); let model_name = format!("model-{token}.json"); fold_model_cmd(prof, model, catalogue, target, class_scope, &p(&model_name))?; artifacts.push(model_name); } // 8. the interop manifest. let manifest = json!({ "version": version, "artifacts": artifacts }); std::fs::write(p("manifest.json"), serde_json::to_string_pretty(&manifest)?)?; eprintln!( "\nproduce {version} -> {} ({} artifacts)", out_dir.display(), artifacts.len() ); Ok(()) } /// Multiset of normalized function-body digests for one image: digest -> how many functions carry it. /// Identical-code-folded clones (many tiny thunks share a body) collapse to one digest with a count >1, /// which the min-count intersection in `classify_change_cmd` then handles exactly. /// /// **Enumerated over the same union `xref::XrefIndex::build` uses, and it has to be.** CS2 strips /// `.eh_frame` from the game code — `.eh_frame_hdr` describes 8,327 of libserver's ~70,000 functions, all /// of them in the statically-linked runtime tail — so digesting the FDE list alone samples the ~12% of the /// binary least likely to change and calls it the whole. A gameplay-only patch then shows zero changed /// digests and `classify-change` answers `skip`: "no release needed", about a build whose gamedata moved. /// Relocation code-pointers ∪ decoded call targets ∪ FDE starts covers the gameplay region too. /// /// A `[start, next_start)` range can span a real function plus an unindexed neighbour, so a digest is not /// a claim about one function. It does not need to be: what both callers compare is the MULTISET, and a /// range that is stable across two builds carries the same digest in both whatever it contains. fn function_digests(img: &CodeImage) -> HashMap { let entries = crate::locate::function_entries(img); let mut m: HashMap = HashMap::new(); for (i, &start) in entries.iter().enumerate() { // The last entry runs to the end of its executable block, which `normalized_digest` clamps to. let end = entries.get(i + 1).copied().unwrap_or(u64::MAX); if let Some(d) = emit::normalized_digest(img, start, end) { *m.entry(d).or_default() += 1; } } m } /// Function-body change between two builds' digest multisets. `common` = body-identical functions /// (the multiset intersection); everything else is derived. Shared by `classify-change` and /// `filter-corpus` so their notion of "changed" is one definition. struct DigestChange { n_prev: u32, n_new: u32, common: u32, } /// The change class between two builds — a closed domain both CI commands decide identically from a /// `DigestChange` (via [`DigestChange::classify`]) and serialize to its lowercase name, so `classify-change` /// and `filter-corpus` can't drift on what "shift" means. #[derive(Clone, Copy, PartialEq, Eq)] enum ChangeVerdict { Skip, Normal, Shift, } impl ChangeVerdict { fn as_str(self) -> &'static str { match self { ChangeVerdict::Skip => "skip", ChangeVerdict::Normal => "normal", ChangeVerdict::Shift => "shift", } } } impl DigestChange { fn changed(&self) -> u32 { self.n_new - self.common // new-side functions with no body-identical prior = added or edited } fn removed(&self) -> u32 { self.n_prev - self.common // prev-side functions gone or edited } fn frac(&self) -> f64 { if self.n_new == 0 { 0.0 } else { self.changed() as f64 / self.n_new as f64 } } /// Classify this change: `Skip` if code-identical (0 changed) or below the skip tolerance, `Shift` if a /// broad codegen change (at/above the shift threshold), else `Normal`. The one rule both CI commands share. fn classify(&self, skip_below: f64, shift_above: f64) -> ChangeVerdict { if self.changed() == 0 || self.frac() < skip_below { ChangeVerdict::Skip } else if self.frac() >= shift_above { ChangeVerdict::Shift } else { ChangeVerdict::Normal } } } fn digest_change(prev: &HashMap, new: &HashMap) -> DigestChange { let common = new .iter() .map(|(d, &nb)| nb.min(prev.get(d).copied().unwrap_or(0))) .sum(); DigestChange { n_prev: prev.values().sum(), n_new: new.values().sum(), common, } } /// Classify the change between two builds of `lib` into `skip` / `normal` / `shift` (see the `Cmd` /// doc). The signal is the fraction of the NEW build's functions whose masked body isn't byte-identical /// to any function in the previous build — added-or-edited functions over the population we'd re-derive /// against. A toolchain shift regenerates codegen everywhere, so that fraction spikes near 1; an ordinary /// patch touches a few percent; an unchanged rebuild is ~0. pub fn classify_change_cmd( prev: &Path, new: &Path, lib: &str, skip_below: f64, shift_above: f64, json: bool, ) -> Result<()> { ensure!( skip_below <= shift_above, "--skip-below ({skip_below}) must be <= --shift-above ({shift_above})" ); let da = function_digests(&load_lib(prev, lib)?); let db = function_digests(&load_lib(new, lib)?); ensure!( !da.is_empty() && !db.is_empty(), "no functions enumerated in one of the builds — refusing to classify rather than report `skip` \ from an empty sample" ); let ch = digest_change(&da, &db); let (n_prev, n_new, common, changed, removed, frac) = ( ch.n_prev, ch.n_new, ch.common, ch.changed(), ch.removed(), ch.frac(), ); // `skip` iff the library is code-identical (0 functions changed) — the common CI case of a buildid // bump that didn't touch server code, where the gamedata provably can't have changed. `--skip-below` // widens this to a small tolerance; by default only exact code-identity skips, so any real patch // re-derives. `shift` = a broad codegen change (toolchain/build-env) — still just a re-derive, but one // where most byte-sigs have drifted so recovery leans on string-anchors + vtable slots. let verdict = ch.classify(skip_below, shift_above); if json { println!( "{}", json!({ "verdict": verdict.as_str(), "changed_fraction": frac, "changed": changed, "common": common, "removed": removed, "functions_prev": n_prev, "functions_new": n_new, }) ); } else { let hint = match verdict { ChangeVerdict::Skip => "no meaningful code change — no release needed", ChangeVerdict::Shift => "toolchain/compiler shift — re-derive (most byte-sigs drifted)", ChangeVerdict::Normal => "ordinary patch — re-derive from the current model", }; println!("verdict: {} ({hint})", verdict.as_str()); println!( "changed: {:.2}% ({changed}/{n_new} new functions not body-identical to prev)", frac * 100.0 ); println!(" body-identical (common): {common}"); println!(" prev-only (removed/edited): {removed}"); println!(" functions: prev {n_prev}, new {n_new}"); } Ok(()) } /// One build kept by the change-aware filter — a code-distinct representative of a run of otherwise /// code-identical builds. #[derive(serde::Serialize)] struct KeptBuild { label: String, date: String, /// Toolchain era (0-based); incremented at every `shift` transition. era: u32, /// Drift class from the PREVIOUS kept build: "normal" | "shift" | absent (the first kept build). #[serde(skip_serializing_if = "Option::is_none")] drift: Option<&'static str>, /// Changed-fraction from the previous kept build (0 for the first). fraction: f64, /// How many later code-identical builds this representative absorbed. collapsed: u32, } /// Stage-1 change-aware corpus filter. Walks a game's builds chronologically, collapses runs of /// code-identical builds (bodies unchanged, only relocations moved — no gamedata delta) to ONE /// representative, labels each surviving transition `normal`/`shift`, and segments the timeline into /// toolchain ERAS (cut at every `shift`). Emits a selection manifest (the code-distinct kept builds, /// each with its era + drift). Lossless for the per-game facts (a collapsed build's hops are identity) /// and the distinct-build set `corpus-model` distills. Same masked-digest change /// signal as `classify-change`, but digesting each build ONCE in parallel (not pairwise-redundant). pub fn filter_corpus_cmd( prof: &GameProfile, corpus: &Path, lib: &str, skip_below: f64, shift_above: f64, out: Option<&Path>, threads: Option, ) -> Result<()> { ensure!( skip_below <= shift_above, "--skip-below ({skip_below}) must be <= --shift-above ({shift_above})" ); let builds = find_builds(prof, corpus)?; // chronological: labels are date-prefixed, sorted ensure!( builds.len() >= 2, "need >=2 builds under {}", corpus.display() ); let nthreads = default_threads(threads); eprintln!( "filter-corpus: digesting {} builds of {lib} ({nthreads} threads) ...", builds.len() ); // Digest each build ONCE, in chronological order (parallel_map preserves input order). let digested: Vec<(String, HashMap)> = parallel_map(&builds, nthreads, |dir| { let dig = find_file(dir, lib, 8) .and_then(|p| CodeImage::load(&p).ok()) .map(|img| function_digests(&img)) .unwrap_or_default(); (label_of(dir), dig) }); let mut kept: Vec = Vec::new(); let mut dropped: BTreeMap = BTreeMap::new(); let mut shifts: Vec = Vec::new(); let mut era = 0u32; let mut unreadable = 0u32; let mut last_kept: Option = None; // index into `digested` of the last kept build for (i, (label, dig)) in digested.iter().enumerate() { if dig.is_empty() { unreadable += 1; // couldn't load/decode — neither kept nor dropped continue; } let Some(lk) = last_kept else { kept.push(KeptBuild { label: label.clone(), date: build_date(label), era: 0, drift: None, fraction: 0.0, collapsed: 0, }); last_kept = Some(i); continue; }; let ch = digest_change(&digested[lk].1, dig); match ch.classify(skip_below, shift_above) { ChangeVerdict::Skip => { // Code-identical to the last kept build — collapse it into that representative. dropped.insert(label.clone(), digested[lk].0.clone()); if let Some(k) = kept.last_mut() { k.collapsed += 1; } } verdict => { if verdict == ChangeVerdict::Shift { era += 1; shifts.push( json!({ "label": label, "date": build_date(label), "fraction": ch.frac() }), ); } kept.push(KeptBuild { label: label.clone(), date: build_date(label), era, drift: Some(verdict.as_str()), fraction: ch.frac(), collapsed: 0, }); last_kept = Some(i); } } } let (n_total, n_kept, n_dropped) = (builds.len(), kept.len(), dropped.len()); let n_eras = if n_kept == 0 { 0 } else { era + 1 }; eprintln!( "filter-corpus: {n_total} builds -> {n_kept} code-distinct ({n_eras} eras, {} shifts), \ {n_dropped} code-identical dropped{}", shifts.len(), if unreadable > 0 { format!(", {unreadable} unreadable") } else { String::new() } ); let manifest = json!({ "corpus": corpus.display().to_string(), "lib": lib, "skip_below": skip_below, "shift_above": shift_above, "builds_total": n_total, "builds_kept": n_kept, "builds_dropped": n_dropped, "builds_unreadable": unreadable, "eras": n_eras, "shifts": shifts, "kept": kept, "dropped": dropped, }); match out { Some(p) => { std::fs::write(p, serde_json::to_string_pretty(&manifest)?)?; eprintln!(" wrote selection manifest -> {}", p.display()); } None => println!("{}", serde_json::to_string_pretty(&manifest)?), } Ok(()) } // ══════════════════════════════════════════════════════════════════════════════════════════════ // §5 · LIVE ORACLE (semantic verification against a running server) // ══════════════════════════════════════════════════════════════════════════════════════════════ /// The schema inheritance chain (class names, offset-0 bases) of `class` + its on-disk vtable slot /// count — the set of classes whose methods a live instance of `class` actually carries, and how many /// vtable slots it has. fn pawn_chain(prof: &GameProfile, img: &CodeImage, class: &str) -> (HashSet, usize) { let classes = schema::enumerate_schema(img); let by_name: HashMap<&str, &schema::SchemaClass> = classes.iter().map(|c| (c.name.as_str(), c)).collect(); let mut chain = HashSet::new(); let mut cur = Some(class); while let Some(name) = cur { if !chain.insert(name.to_string()) { break; } cur = by_name.get(name).and_then(|c| c.primary_base()); } let n = rtti::find_vtable(img, class, prof.max_vtable_slots).map_or(0, |vt| vt.slots.len()); (chain, n) } /// The live-pawn context a semantic sweep reads and calls against: the process, its on-disk libserver /// image + load slide, one live instance, and that instance's vtable slot count. struct PawnProbe<'a> { pid: u32, live: &'a live::LiveProcess, server_img: &'a CodeImage, base: u64, pawn: u64, vt_slots: usize, } /// Semantic CALL sweep: the composition of the ABI shape (part a) with live calling (part b). For /// every derived vtable-method offset the pawn actually carries whose ABI shape is `this`-only and /// whose name reads as a pure query, CALL `vtable[offset](pawn)` via ptrace and confirm it returns /// cleanly. This turns the single hand-picked IsPlayerPawn probe into a sweep over every safely- /// callable derived method — the semantic gate for a wrong/absent offset. A faulting call is caught /// and the process restored (call_remote suppresses the signal), so a bad offset degrades to a FLAG, /// never a crash. It is a smoke test, not an identity proof: a clean return proves the slot is a /// callable this-method, while the OFFLINE abi-diff proves the prototype itself didn't move. fn callable_method_sweep( prof: &GameProfile, p: &PawnProbe, gamedata: &GdMap, chain: &HashSet, ) -> Result { // A transient read failure here is a property of the live process, not of the derived release. Aborting // would propagate out of `run_live_oracle` and past `produce`'s fail-fast, throwing away a fully derived // in-memory release over one unreadable address — so treat an unreadable pawn/slot the way every other // per-slot failure in this loop is treated (and the way `fuzz_live_run` already does): record and move on. let Ok(vtable_ptr) = p.live.read_u64(p.pawn) else { eprintln!( " sweep SKIPPED: pawn {:#x} vtable unreadable (object freed mid-sweep?)", p.pawn ); return Ok(OracleCounts::default()); }; let (mut attempted, mut clean, mut not_this_only, mut not_query) = (0u32, 0u32, 0u32, 0u32); let mut faulted: Vec = Vec::new(); // Deterministic order: gamedata is a key-sorted map, iterate it directly. for (name, entry) in gamedata { let Some(off) = render::entry_from_value(entry).offset else { continue; // not a vtable-method offset (a signature or field) }; if off < 0 || (off as usize) >= p.vt_slots || !chain.contains(class_of(name)) { continue; // not a method this pawn carries at a valid slot } if !is_query_method(prof, name) { not_query += 1; continue; } let Ok(func) = p.live.read_u64(vtable_ptr + off as u64 * 8) else { faulted.push(format!("{name} — slot {off} unreadable")); continue; }; if !p.live.is_exec(func) { faulted.push(format!("{name} — slot {off} not executable")); continue; } // Shape the target from the on-disk image (file vaddr = runtime - slide); only invoke a // this-only method (int_args<=1, no float/stack args) — anything else needs arguments we // don't have, so calling it would pass garbage. let this_only = abi::abi_shape(p.server_img, func.wrapping_sub(p.base)) .is_some_and(|s| s.is_this_only()); if !this_only { not_this_only += 1; continue; } attempted += 1; match live::call_remote(p.pid as i32, func, &[p.pawn]) { Ok(r) if r.clean_return => clean += 1, Ok(_) => faulted.push(format!("{name} — offset {off} faulted (wrong offset?)")), Err(e) => faulted.push(format!("{name} — call errored: {e}")), } } println!( " called {clean}/{attempted} this-only query methods cleanly on the live pawn \ ({not_this_only} need args, {not_query} not query-safe — skipped)" ); for f in faulted.iter().take(40) { println!(" FLAG {f}"); } if faulted.len() > 40 { println!(" … and {} more", faulted.len() - 40); } Ok(OracleCounts { checked: attempted, ok: clean, faulted: faulted.len() as u32, }) } /// Deterministic 64-bit LCG (standard Numerical-Recipes constants) — reproducible randomized sequencing for the /// live fuzzer, no `rand` dep. A found crash replays exactly given the same `--seed`. struct Lcg(u64); impl Lcg { fn next(&mut self) -> u64 { self.0 = self .0 .wrapping_mul(6364136223846793005) .wrapping_add(1442695040888963407); // Return the HIGH bits. In a power-of-two-modulus LCG, bit k has period 2^(k+1), so the low bits // are nearly constant: returning them would make `below(2)` — the call-vs-netvar coin — a period-4 // sequence, and an iteration over a class with both methods and fields draws exactly 4 times, which // would lock the coin to one value across consecutive such iterations and starve one probe kind. self.0 >> 33 } fn below(&mut self, n: usize) -> usize { if n == 0 { 0 } else { (self.next() % n as u64) as usize } } } /// A live class the fuzzer can probe: how to find its instances (on-disk vtable slot0 → runtime via /// slide), its safely-callable this-only query methods (from the derived gamedata), and its schema /// fields (for netvar reads). struct LiveClass { vtable_slot0: u64, query_methods: Vec<(String, usize)>, // (gamedata name, vtable slot) fields: Vec<(String, i32)>, // (netvar, offset) } /// What a live oracle stage actually concluded, as data rather than as stdout. The stages compute a /// verdict and then print it; returning it lets `run_live_oracle` gate on the numbers and lets CI branch /// on them, which is the whole point of running the checks. Absent this, the netvars half of a release /// was governed by nothing but a human reading a log line. #[derive(Default, Clone, Copy)] pub(crate) struct OracleCounts { pub(crate) checked: u32, pub(crate) ok: u32, pub(crate) faulted: u32, } impl OracleCounts { /// Fraction of checks that passed, 1.0 when nothing was checked (a stage that had nothing to do is /// not a failure — a stage that checked things and mostly failed is). fn pass_rate(&self) -> f64 { if self.checked == 0 { 1.0 } else { self.ok as f64 / self.checked as f64 } } } /// Minimum share of live checks that must pass before a release is allowed through. Set below the /// clean-run pass rate, not at it: the sweep legitimately faults on methods whose real prototype needs /// arguments, and the schema walk races entities being freed. This is a floor that catches "the layout /// moved and nearly everything now fails", not a target to tune. A clean run passes near 100% on both /// the schema-layout and call-sweep stages, so 0.90 leaves real headroom. const ORACLE_MIN_PASS: f64 = 0.90; /// Below this many checks, a pass RATE is noise and the floor is not applied — the stage still reports. /// The call sweep legitimately checks only a handful of methods (most vtable entries either need /// arguments or aren't query-safe), where a single transient fault is a large fraction and would block /// an otherwise clean release. Rate gating needs a denominator big enough to mean something; the wide /// schema walk is the case this floor is really for. const ORACLE_MIN_SAMPLE: u32 = 25; /// Minimum fraction of schema fields that must resolve to a runtime TYPE before the typed netvars ship. /// `live_schema` reads each field's type through a SECOND runtime struct (`CSchemaType`, via F_TYPE/TY_NAME/ /// TY_CATEGORY) that the schema-layout oracle never exercises, defaulting to empty on any failure. If a build /// moves those type-record offsets, every field resolves 'untyped', the layout gate still passes, and a /// netvars file full of empty types would ship at exit 0 — the one thing the live walk adds over the offline /// reader, ungated. Normal runs resolve ~100% typed (0 untyped observed), so a 0.5 floor only ever trips on a /// wholesale type-record reshape, not on the odd unresolved field. const NETVARS_MIN_TYPED: f64 = 0.5; /// The live-fuzzing loop against an ALREADY-ATTACHED server — shared by `produce`'s live stage and the /// `integration-test` harness. Both own the server already, so there is no separate launch and no fixed /// wall-clock: it runs exactly `iterations` probes and stops. /// /// `seed` is passed FIXED (`0x5137`) by both callers and no flag varies it, so every run replays the same /// probe sequence. That is deliberate for a release gate — a failure is reproducible by re-running the /// same command — and it is the reason this is a smoke test rather than a search. fn fuzz_live_run( prof: &GameProfile, pid: u32, img: &CodeImage, live: &live::LiveProcess, base: u64, doc: &GdMap, iterations: usize, seed: u64, ) -> Result<()> { // Group derived vtable-method offsets by class; keep the this-only query methods of each class that // has an RTTI vtable, plus that class's schema fields. let schema_by_name: HashMap> = schema::enumerate_schema(img) .into_iter() .map(|c| { ( c.name, c.fields.into_iter().map(|f| (f.name, f.offset)).collect(), ) }) .collect(); let mut by_class: BTreeMap<&str, Vec<(String, usize)>> = BTreeMap::new(); for (name, entry) in doc { if !is_query_method(prof, name) { continue; } if let Some(off) = render::entry_from_value(entry).offset && off >= 0 { by_class .entry(class_of(name)) .or_default() .push((name.clone(), off as usize)); } } let mut classes: Vec = Vec::new(); for (cname, methods) in by_class { let Some(vt) = rtti::find_vtable(img, cname, prof.max_vtable_slots) else { continue; }; // keep only this-only methods (int_args<=1, no float/stack) at valid slots let query_methods: Vec<(String, usize)> = methods .into_iter() .filter(|(_, slot)| { *slot < vt.slots.len() && vt .slots .get(*slot) .is_some_and(|&a| abi::abi_shape(img, a).is_some_and(|s| s.is_this_only())) }) .collect(); let fields = schema_by_name.get(cname).cloned().unwrap_or_default(); if query_methods.is_empty() && fields.is_empty() { continue; } // No `vt_slots` here: `query_methods` is already filtered to `slot < vt.slots.len()` above, so // carrying the bound only to re-test it downstream reads as a guard that can fire when it cannot. classes.push(LiveClass { vtable_slot0: vt.slot0, query_methods, fields, }); } // A SMOKE TEST with nothing to probe is a smoke test that found no problems. Aborting here would // fail a release that is already fully derived and live-validated, on the grounds that an optional // extra check had no sample — the opposite of "degrade or stop loudly": the loud part is right, the // stopping is not. Zero probeable classes is a real thing to say, so say it and return. if classes.is_empty() { eprintln!( " fuzz-live SKIPPED: no probeable classes (no gamedata offset resolves to a class with an \ RTTI vtable and a this-only query method). Nothing was fuzzed; nothing failed." ); return Ok(()); } let n_methods: usize = classes.iter().map(|c| c.query_methods.len()).sum(); eprintln!( "fuzz-live: {} probeable classes, {n_methods} this-only query methods, {} iterations, seed {seed}", classes.len(), iterations ); let mut rng = Lcg(seed); let (mut calls, mut clean, mut reads, mut readable, mut no_inst) = (0u64, 0u64, 0u64, 0u64, 0u64); let mut faults: BTreeMap = BTreeMap::new(); // probe -> fault count for it in 0..iterations { let c = &classes[rng.below(classes.len())]; let instances = live.find_instances(c.vtable_slot0 + base, 16); if instances.is_empty() { no_inst += 1; continue; } let inst = instances[rng.below(instances.len())]; // half the time call a method, half read a netvar (whichever is available) let do_call = !c.query_methods.is_empty() && (c.fields.is_empty() || rng.next() & 1 == 0); if do_call { let (name, slot) = &c.query_methods[rng.below(c.query_methods.len())]; let Ok(vtp) = live.read_u64(inst) else { continue; }; let Ok(func) = live.read_u64(vtp + *slot as u64 * 8) else { continue; }; if !live.is_exec(func) { *faults.entry(format!("{name} (slot not exec)")).or_default() += 1; continue; } calls += 1; match live::call_remote(pid as i32, func, &[inst]) { Ok(r) if r.clean_return => clean += 1, Ok(_) => *faults.entry(name.clone()).or_default() += 1, Err(e) => { // a vanished target may mean the server crashed — confirm by re-attaching. if live::LiveProcess::attach(pid).is_err() { eprintln!( " !! server pid {pid} appears DEAD after calling {name} on {inst:#x} at iteration {it} — possible crash (seed {seed})" ); anyhow::bail!("live server died during fuzzing (last probe: {name})"); } *faults.entry(format!("{name} (call err: {e})")).or_default() += 1; } } } else if !c.fields.is_empty() { let (_fname, off) = &c.fields[rng.below(c.fields.len())]; reads += 1; if *off >= 0 && live.read_i32(inst + *off as u64).is_ok() { readable += 1; } } } println!("fuzz-live done ({iterations} iterations):"); println!(" method calls : {clean}/{calls} returned cleanly"); println!(" netvar reads : {readable}/{reads} readable"); println!(" no-instance iterations (class had none live at the time): {no_inst}"); if faults.is_empty() { println!(" no faults — every derived probe held across the sampled game states."); } else { println!( " {} probe(s) FAULTED (offset may be wrong, or state-dependent):", faults.len() ); for (probe, n) in faults.iter().take(40) { println!(" {n:>4}x {probe}"); } } Ok(()) } pub fn integration_test_cmd( prof: &GameProfile, game: &Path, build: Option<&Path>, lib: &str, wait: u64, map: &str, bots: u32, gamedata: Option<&Path>, out: Option<&Path>, keep: bool, fuzz_iterations: usize, ) -> Result<()> { let build = build.unwrap_or(game); // Read the gamedata to validate against into memory (run_live_oracle takes it as a string). let gamedata = gamedata.map(std::fs::read_to_string).transpose()?; // Launch the vanilla bots server (shared with produce) and run the identical live oracle on it. let mut server = launch_bots_server(prof, game, build, lib, wait, map, bots)?; let pid = server.pid; let result = run_live_oracle(prof, pid, build, lib, gamedata.as_deref(), fuzz_iterations); if keep { eprintln!("\n(leaving server pid {pid} running as requested)"); } else { let _ = server.child.kill(); let _ = server.child.wait(); eprintln!("\n(test server pid {pid} shut down)"); } // Surface any oracle error only AFTER teardown; then, if requested, write the kept (live-valid) set. let live_result = result?; if let (Some(op), Some((kept, _))) = (out, &live_result) { std::fs::write( op, serde_json::to_string_pretty(&Value::Object(kept.clone()))?, ) .with_context(|| format!("write {}", op.display()))?; eprintln!(" wrote {} kept entries -> {}", kept.len(), op.display()); } Ok(()) } /// Resolve the live-oracle readiness anchor from the on-disk image: for a pawn game, the player-pawn /// vtable + the health-netvar offset (poll for an ALIVE pawn); for a pawn-less game (Dota), the /// gamerules-proxy vtable (a live instance = map loaded + libserver ready) and no health field. fn resolve_ready_anchor( prof: &GameProfile, img: &CodeImage, ) -> Result<(rtti::VTable, Option)> { match prof.pawn_anchor { Some(pa) => { let health = schema::enumerate_schema(img) .iter() .flat_map(|c| &c.fields) .find(|f| f.name == pa.health_field) .map(|f| f.offset as u64) .context("health field offset")?; let vt = rtti::find_vtable(img, pa.pawn_class, prof.max_vtable_slots) .context("pawn vtable")?; Ok((vt, Some(health))) } None => { let cls = prof.ready_class; let vt = rtti::find_vtable(img, cls, prof.max_vtable_slots) .with_context(|| format!("{cls} vtable"))?; Ok((vt, None)) } } } /// The live-oracle pawn context: an alive player pawn plus the anchor and health-field offset needed to CALL /// and sweep it. `Some` only when all three inputs are present (a pawn game with an alive pawn found); a /// pawn-less game (Dota) or a not-yet-spawned pawn leaves it `None`, gating the pawn-only CALL test + sweep. #[derive(Clone, Copy)] struct PawnContext { anchor: profile::PawnAnchor, pawn: u64, health: u64, } pub(crate) fn run_live_oracle( prof: &GameProfile, pid: u32, build: &Path, lib: &str, gamedata: Option<&str>, fuzz_iterations: usize, ) -> Result>)>> { let pawn_anchor = prof.pawn_anchor; let img = load_lib(build, lib)?; let (ready_vt, pawn_health) = resolve_ready_anchor(prof, &img)?; let live = live::LiveProcess::attach(pid)?; let base = live .base(lib) .with_context(|| format!("{lib} not mapped in pid {pid}"))?; // The server is already ready (the caller launched + polled it); for a pawn game, grab the alive pawn. let pawn = match pawn_health { Some(health) => live .find_instances(ready_vt.slot0 + base, 64) .into_iter() .find(|&o| live.read_i32(o + health).unwrap_or(0) > 0), None => None, }; // Group the three pawn Options into one context — present iff all three are (and `pawn` is only `Some` // when `pawn_health` is, so the context is `Some` exactly when a pawn game has an alive pawn). let pawn_ctx = match (pawn_anchor, pawn, pawn_health) { (Some(anchor), Some(pawn), Some(health)) => Some(PawnContext { anchor, pawn, health, }), _ => None, }; // The derived gamedata, parsed ONCE: the CALL test below needs THIS build's IsPlayerPawn slot, and // the validate stage needs the whole document. let doc = gamedata.map(read_gamedata_str).transpose()?; eprintln!("\n=== read-only oracle on the owned process ==="); let mut verdicts: Vec<(&str, OracleCounts)> = Vec::new(); verdicts.push(("schema-layout", verify_live_cmd(prof, pid, build, lib)?)); // The CALL test needs a live player pawn; a pawn-less game (Dota) runs only the schema oracle above // and the sig validation below. if let Some(PawnContext { anchor: pa, pawn, health, }) = pawn_ctx { // A closure so a "nothing to test" exit skips the CALL test ALONE — the Pulse shim and descriptor // oracles below are independent of it and must still run. (|| { println!("\n=== CALL test (ptrace injection — the thing read-only can't do) ==="); // The slot THIS build derived, never the constant frozen in the profile. `IsPlayerPawn` is an // offset-only name, so `derive_offsets` drops it into `unresolved` whenever no anchor chains or // the recency-weighted vote falls under the bar — and it does that precisely in the builds where // the slot MOVED. Falling back to the frozen index there would be the worst possible moment: // the catalogue shows this slot taking six distinct values in ten months, and the fallback would // ptrace-CALL whatever now occupies a stale index, on the same process this run goes on to read // typed netvars from and fuzz 500 times, then print PASS about an offset the gamedata lacks. // `pa.is_player_pawn_slot` is therefore a RECORDED REFERENCE VALUE, cross-checked against and // never itself called. let derived = doc .as_ref() .and_then(|d| d.get("CBaseEntity::IsPlayerPawn")) .and_then(|e| render::entry_from_value(e).offset) .and_then(|o| u64::try_from(o).ok()); // The two ways of having no slot are different facts and get different sentences: an // `integration-test` run with no `--gamedata` is normal, while a `produce` run that derived // nothing is a statement about this build. One WARNING covering both blamed the derivation for // the perfectly ordinary case. let Some(is_player_pawn) = derived else { match doc { None => println!( " no gamedata document supplied, so there is no derived slot to test — skipping. \ (`integration-test` without --gamedata; `produce` always supplies one.)" ), Some(_) => eprintln!( " WARNING this build derived NO IsPlayerPawn slot — it is offset-only, so it lands \ in `unresolved` when no anchor chains or the vote is under the bar. SKIPPING the \ CALL test rather than injecting a call at the profile's recorded {}. The semantic \ sweep below still covers every offset this build DID derive.", pa.is_player_pawn_slot ), } return; }; if is_player_pawn != pa.is_player_pawn_slot { eprintln!( " NOTE derived IsPlayerPawn slot {is_player_pawn} differs from the profile's recorded \ {} — using the derived one; update GameProfile::is_player_pawn_slot", pa.is_player_pawn_slot ); } // A pawn selected back at pawn-probe time can be freed during the wide schema walk that ran in // between (a bots-deathmatch pawn dies and is unmapped). That is a property of the live process, // not of the fully-derived in-memory release — so a read failure here must DEGRADE rather than // `?`-propagate past produce's fail-fast and abort. Each bail names the reason it actually hit: // one message said "pawn became unreadable" for every exit, including the one where the pawn read // fine and the SLOT was wrong, which reframed a stale offset as a benign live-process race. let outcome = (|| -> Result<(), String> { let hp = live .read_i32(pawn + health) .map_err(|_| format!("pawn {pawn:#x} became unreadable (freed mid-oracle?)"))?; println!("alive pawn {pawn:#014x}, live m_iHealth = {hp}"); let vtable_ptr = live .read_u64(pawn) .map_err(|_| format!("pawn {pawn:#x} has no readable vtable pointer"))?; let func = live .read_u64(vtable_ptr + is_player_pawn * 8) .map_err(|_| { format!("slot {is_player_pawn} is not readable in the live vtable") })?; // Same gate the other two `call_remote` sites apply: never inject a call to something that // is not executable code in the live process. if !live.is_exec(func) { return Err(format!( "slot {is_player_pawn} does not point at live executable code — the derived offset \ is wrong or stale, which is a fact about the BUILD, not about this process" )); } println!( "calling IsPlayerPawn (gamedata vtable offset {is_player_pawn}, fn {func:#x}) on the live pawn..." ); let r = live::call_remote(pid as i32, func, &[pawn]) .map_err(|e| format!("the call itself could not be made: {e:#}"))?; let ret = r.rax & 0xff; println!( " -> returned {ret} (expect 1=true), clean_return={}", r.clean_return ); if r.clean_return && ret == 1 { println!( " PASS: gamedata offset {is_player_pawn} semantically IS IsPlayerPawn — verified by CALLING it." ); } else { println!( " (unexpected result — offset may be wrong, or the pawn wasn't a player pawn)" ); } Ok(()) })(); if let Err(why) = outcome { eprintln!(" CALL test SKIPPED: {why}"); } })(); } // The Pulse shim contract, checked by calling. Independent of `--gamedata`: it verifies a claim // the binding registry makes, not a gamedata locator, so it runs on every live oracle. println!( "\n=== Pulse invocation shims: calling every `args-only` binding with a sentinel handle ===" ); // Read once, used by both verifiers below — see `PulseRead`. let pulse = PulseRead::of(&img, 8); let (shims_probed, shims_ok, shims_faulted, shim_notes) = verify_pulse_shims(&live, pid, base, &img, &pulse); println!(" {shims_ok} / {shims_probed} returned cleanly"); for n in shim_notes.iter().take(8) { println!(" {n}"); } // Probing NOTHING must not read as a pass. `OracleCounts::pass_rate` returns 1.0 for zero checks — // correct in general, since a stage with nothing to do is not a failure — but here zero means the // eligibility filter stopped matching, which is precisely the silent collapse the emitted // `call.needs` field would then be making claims about. The profile floor guarantees the callable // rows exist, so an empty probe set is a contradiction worth shouting about. if shims_probed == 0 { println!( " WARNING: no shim was eligible to probe. The floor guarantees host-callable rows exist, so \ this means the probe's own filter no longer matches them — the emitted `call.needs` is \ UNVERIFIED for this build." ); } // A fault means the emitted argument contract is wrong, which is a claim the artifact should not be // making. Anything else (a clean non-`-2` return) is a different status protocol, not a broken contract, // so it counts as OK. verdicts.push(( "pulse-shims", OracleCounts { checked: shims_probed as u32, ok: shims_ok as u32, // COUNTED where the fault happens, not reconstructed by grepping the code's own prose for // the word it printed — an edit to that sentence used to silently change this number. faulted: shims_faulted as u32, }, )); // The reconstructed Pulse signatures, against the descriptors the server actually holds. Like the // shim check this verifies a claim the binding registry makes rather than a gamedata locator, so it // runs on every live oracle. println!( "\n=== Pulse descriptors: the reconstructed signature vs the one the live server holds ===" ); let (desc_checked, desc_agree, desc_notes) = verify_pulse_descriptors(&live, pid, base, &img, &pulse); println!(" {desc_agree} / {desc_checked} agree"); for n in desc_notes.iter().take(8) { println!(" {n}"); } if desc_checked == 0 { println!( " WARNING: no descriptor region could be read. Either the accessors stopped populating on \ call or the region layout moved — either way the emitted `params` are UNVERIFIED for this \ build." ); } verdicts.push(( "pulse-descriptors", OracleCounts { checked: desc_checked as u32, ok: desc_agree as u32, faulted: (desc_checked - desc_agree) as u32, }, )); let live_result = if gamedata.is_some() { println!("\n=== validate-live: derived gamedata vs the running server ==="); // Parsed once, above — it feeds the CALL test's slot, sig/offset validation, and the pawn // sweep/fuzz below. let doc = doc.expect("parsed above whenever `gamedata` is Some"); let (kept, entry_verdicts, val_counts) = validate_live_cmd(prof, pid, build, &doc)?; verdicts.push(("validate-live", val_counts)); // The semantic sweep + live fuzz operate on a live pawn; pawn-less games stop at sig validation. if let Some(PawnContext { anchor: pa, pawn, .. }) = pawn_ctx { let (chain, vt_slots) = pawn_chain(prof, &img, pa.pawn_class); println!( "\n=== semantic call sweep (every this-only query method the pawn carries) ===" ); let probe = PawnProbe { pid, live: &live, server_img: &img, base, pawn, vt_slots, }; verdicts.push(( "call-sweep", callable_method_sweep(prof, &probe, &doc, &chain)?, )); if fuzz_iterations > 0 { println!( "\n=== live fuzz: {fuzz_iterations} randomized probes across live game state ===" ); fuzz_live_run(prof, pid, &img, &live, base, &doc, fuzz_iterations, 0x5137)?; } } Some((kept, entry_verdicts)) } else { None }; // Gate on what the stages concluded: a stage that only printed its verdict would leave a run where the // schema layout moved and nearly every class mismatched indistinguishable, to the caller and to CI, // from a clean one. for (stage, c) in &verdicts { eprintln!( " oracle {stage}: {}/{} ok, {} faulted{}", c.ok, c.checked, c.faulted, if c.checked < ORACLE_MIN_SAMPLE { " (below gating sample — reported only)" } else { "" } ); ensure!( c.checked < ORACLE_MIN_SAMPLE || c.pass_rate() >= ORACLE_MIN_PASS, "live oracle stage `{stage}` passed only {}/{} checks ({:.1}%, floor {:.0}%) — the runtime \ layout likely moved; refusing to publish a release derived against it", c.ok, c.checked, c.pass_rate() * 100.0, ORACLE_MIN_PASS * 100.0 ); } Ok(live_result) } /// The sentinel entity handle the Pulse resolve preamble rejects before dereferencing anything. const PULSE_INVALID_HANDLE: u32 = 0xffff_ffff; /// `PVAL_EHANDLE`, from the shipped `PulseValueType_t`. const PULSE_EHANDLE: i32 = 13; /// The owner-pointer offset within a VScript record. The STRIDE is `vscript::STRIDE` — the offline /// reader's own, rather than a second copy of the same number that would have to be kept in step. const VS_OWNER: u64 = 0x30; /// Attribute every VScript binding to the class that owns it, from the running server. /// /// **This is the one VScript field that cannot be derived offline**, and it is worth being precise about /// why. The record stores its owner at `+0x30`, but the class descriptor reaches the initialiser through /// a register loaded from memory rather than a `lea`, so constant propagation recovers the pointer for /// exactly zero of the 1,841 bindings. Nothing about the fold can fix that; the value is not in the /// instruction stream. /// /// A running server has it. Every record in one array points at the same descriptor, and the descriptor's /// own `+0x00` is its class-name string — the same 16-byte name-pair shape the records use. Walking that /// chain attributes 1,724 of 1,809 members to 66 classes on Dota. /// /// The precedent for a live-only field is the typed schema itself: a schema field's TYPE is a null /// placeholder on disk and is populated only at runtime, which is why an offline run ships no typed /// netvars at all. `class` behaves the same way — present in a full build, absent from an offline one — /// rather than inventing a new kind of gap. /// /// It matters because the consumers group by class. Both `api.json` and the `.d.ts` the Dota ecosystem /// publishes declare members under their owning interface, so a flat function list is not renderable into /// either. fn attribute_vscript_classes( live: &live::LiveProcess, base: u64, img: &CodeImage, bindings: &mut model::Bindings, ) -> (usize, usize) { let ident = |s: &str| { !s.is_empty() && s.len() <= 128 && s.chars() .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == ':') && s.chars() .next() .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') }; let name_at = |addr: u64| -> Option { live.read_u64(addr) .ok() .filter(|&p| p != 0) .and_then(|p| live.read_cstr(p).ok()) .filter(|s| ident(s)) }; let mut of_member: HashMap = HashMap::new(); // Anchor on a name the fold already recovered, then walk that record's whole array. Most anchors land // in an array a previous one covered, so the skip keeps this O(classes) rather than O(bindings). for f in crate::vscript::vscript_functions(img) { if of_member.contains_key(&f.name) { continue; } let mut needle = f.name.clone().into_bytes(); needle.push(0); let Some(&sv) = img.find_bytes(&needle).first() else { continue; }; for h in live.find_instances(base + sv, 32) { let Ok(owner) = live.read_u64(h + VS_OWNER) else { continue; }; let Some(cname) = (owner != 0).then(|| name_at(owner)).flatten() else { continue; }; // Members of one class are contiguous, and a change of owner is the array boundary — the // only honest stopping condition. Walking until a record merely fails to validate over-runs // into whatever is adjacent and attributes unrelated structures to the class. for dir in [-1i64, 0, 1] { let mut k = 0i64; while let Some(addr) = h.checked_add_signed(k.wrapping_mul(crate::vscript::STRIDE)) { if live.read_u64(addr + VS_OWNER).ok() != Some(owner) { break; } if let Some(m) = name_at(addr) { of_member.entry(m).or_insert_with(|| cname.clone()); } if dir == 0 || k.abs() > 4096 { break; } k += dir; } } } } // Count the classes ACTUALLY ASSIGNED, not the ones the walk encountered. Those differ: a class reached // through a late anchor whose members a previous anchor already claimed is encountered and assigned // nothing (`of_member` is first-writer-wins), so the encountered count drifts run to run with which // instances the memory search happened to find — 63 on one Dota run and 66 on the next, both attributing // an identical 1,638 bindings. The assigned count is the stable one and is also the number a consumer // cares about: `gen`'s `moddota` format groups by class, so it is exactly how many interfaces // they will emit. let mut set = 0usize; let mut assigned: HashSet<&str> = HashSet::new(); for b in &mut bindings.vscript { if let Some(c) = of_member.get(&b.name) { b.class = Some(c.clone()); set += 1; } } assigned.extend(bindings.vscript.iter().filter_map(|b| b.class.as_deref())); bindings.meta.vscript_classed = set; (set, assigned.len()) } /// Offset of the name POINTER within a Pulse descriptor element. /// /// MEASURED against a populated region rather than taken from the reconstruction: `+0x00` holds a hash /// token of the name, the string pointer is at `+0x08`, and the type follows at `+0x10`. The offline /// reader never needs this because it collects identifier stores anywhere in the region and assigns them /// by stride; a live read does, and assuming `+0x00` yields a page of spurious disagreements. /// /// The element STRIDE is deliberately not a constant here. `pulse::read_all` derives it per image by /// consensus and the fold reports it, so that a build which resizes the record shows up as a new number /// rather than as lost signatures — a constant would make the offline reader adapt while this oracle /// silently read the wrong addresses, and report a clean verdict for doing so. const PULSE_ELEM_NAME: u64 = 0x08; /// One read of the Pulse registry, shared by both live verifiers. /// /// They opened with the same three lines and each re-walked the whole image for a registry the other had /// already built. Reading it once also means both see the SAME derived stride, which is the property that /// matters: two verifiers disagreeing about the record layout would disagree about which release is good. struct PulseRead { regs: Vec, sigs: Vec>, /// Element spacing, derived per image by consensus. `0` = no record answered unambiguously, which /// `pulse::params_at` already handles by dropping every multi-element list. stride: u64, } impl PulseRead { fn of(img: &CodeImage, threads: usize) -> Self { let regs = crate::valvetab::pulse_bindings(img); let pairs: Vec<(u64, u64)> = regs .iter() .map(|b| (b.descriptor, b.arg_descriptor)) .collect(); let (sigs, stride, _, _) = crate::pulse::read_all(img, &pairs, threads); PulseRead { regs, sigs, stride } } } /// Check the reconstructed Pulse signatures against the descriptors the RUNNING server actually holds. /// /// The Pulse signatures in `surfaces.pulse` are a constant-propagated reconstruction of an initialiser, /// not a read of data — on disk the descriptor elements are zeroes, because they are written at runtime. /// Nothing offline can confirm that reconstruction, which is why the multi-library duplicate check has /// been able to report that the registry disagrees with itself (CS2 331 of 419 repeat registrations, /// Dota 271 of 359) with no way to say which account was right. /// /// A live server can say. The regions are plain statics, so at `slide + base` the real elements are /// there — **once something has called the accessor.** They are lazy-init singletons and a server that /// never executes a Pulse graph leaves every one of them zeroed, which is the normal case: a standard /// match executes none. So the oracle calls them, and that is the whole trick. /// /// **Why calling them is safe.** These are the `+24`/`+32` accessors the fold deliberately REFUSES to /// treat as locators — nullary, measured `int=0 float=0`, whose entire body builds a static once and /// returns it. They take no arguments to get wrong and touch no game state. The same objects that are /// worthless as locators are exactly what makes this check possible. fn verify_pulse_descriptors( live: &live::LiveProcess, pid: u32, base: u64, img: &CodeImage, pulse: &PulseRead, ) -> (usize, usize, Vec) { let (regs, sigs) = (&pulse.regs, &pulse.sigs); let (mut checked, mut agree) = (0usize, 0usize); let mut notes: Vec = Vec::new(); for (b, sig) in regs.iter().zip(sigs.iter()) { let Some(sig) = sig.as_ref() else { continue; }; let Some((region, count)) = crate::pulse::record_region(img, b.descriptor) else { continue; }; // Populate the singleton. A failure here is not a defect in the artifact — it means the call did // not land — so it drops out of the sample rather than counting against agreement. if live::call_remote(pid as i32, base + b.descriptor, &[]).is_err() { continue; } let mut live_names: Vec = Vec::new(); let mut readable = true; for i in 0..count { let e = base + region + i * pulse.stride; match live .read_u64(e + PULSE_ELEM_NAME) .ok() .filter(|&p| p != 0) .and_then(|p| live.read_cstr(p).ok()) { Some(n) => live_names.push(n), None => { readable = false; break; } } } if !readable { continue; } let offline: Vec = sig.args.iter().map(|p| p.name.clone()).collect(); checked += 1; if offline == live_names { agree += 1; } else if notes.len() < 8 { notes.push(format!( "{}: derived {offline:?} but the live descriptor holds {live_names:?}", b.name )); } } (checked, agree, notes) } /// Verify the emitted Pulse invocation shims by CALLING them — with a handle the engine must reject. /// /// The binding registry states that a shim whose `call.needs` is `args-only` can be invoked by a host. /// That is a claim about behaviour, so it is checked against behaviour rather than left as a derivation: /// each eligible binding is called with a sentinel handle, and its resolve must return `-2` without /// dereferencing anything. Confirming the ARGUMENT CONTRACT (the array at `r8+8+8k`, nulls in the slots the /// measurement says are unread) is the point; the sentinel is what makes it free of side effects. /// /// **Why this is safe to run in CI.** Every slot but the argument array is null, so a shim that misuses one /// dereferences null and FAULTS — and a fault is caught, the signal suppressed and the thread restored. The /// dangerous case is a valid-but-wrong pointer, which corrupts silently (see [`crate::taxonomy`]); this /// passes none. The argument array points into the call's own dead stack scratch. /// /// Eligibility is narrow on purpose: a shim, `args-only`, no declared return (so the output sink is never /// needed), and a leading `PVAL_EHANDLE` (so the sentinel is rejected). Anything else is not probed. /// /// Derived from the image with the same readers the fold uses, rather than read back from /// the binding registry: the oracle runs for `integration-test` too, which never builds one, and /// threading it through both callers to re-parse hex strings would verify the same claim by a longer route. fn verify_pulse_shims( live: &live::LiveProcess, pid: u32, base: u64, img: &CodeImage, pulse: &PulseRead, ) -> (usize, usize, usize, Vec) { let (regs, sigs) = (&pulse.regs, &pulse.sigs); let mut probed = 0usize; let mut bailed = 0usize; let mut faulted = 0usize; let mut bad: Vec = Vec::new(); for (b, sig) in regs.iter().zip(sigs.iter()) { let (Some(sig), true) = (sig.as_ref(), b.shim != 0) else { continue; }; let name = &b.name; let callable = crate::pulse::shim_reads(img, b.shim).is_some_and(|r| r.needs() == "args-only"); if !callable || !sig.returns.is_empty() || sig.args.first().map(|p| p.ty) != Some(PULSE_EHANDLE) || sig.args.len() > 2 { continue; } let at = base + b.shim; if !live.is_exec(at) { continue; } // [0x00] padding — the array is addressed from +8 and nothing reads +0 // [0x08] pointer to argument 0 -> relocated to 0x20 // [0x10] pointer to argument 1 -> relocated to 0x24 // [0x20] the sentinel handle, [0x24] a zero second argument let mut blob = [0u8; 0x28]; blob[0x20..0x24].copy_from_slice(&PULSE_INVALID_HANDLE.to_le_bytes()); let relocs: &[(usize, i64)] = if sig.args.len() >= 2 { &[(0x08, 0x20), (0x10, 0x24)] } else { &[(0x08, 0x20)] }; let args = [ live::Arg::Val(0), live::Arg::Val(0), live::Arg::Val(0), live::Arg::Val(0), live::Arg::Scratch(0), live::Arg::Val(0), ]; probed += 1; match live::call_remote_ex( pid as i32, at, &args, &[], Some(live::Scratch { bytes: &blob, relocs, }), ) { Ok(r) if r.clean_return && r.rax as i32 == -2 => bailed += 1, Ok(r) if r.clean_return => { // A clean return that is not the bail path still proves the contract; only note it. bailed += 1; if bad.len() < 8 { bad.push(format!( "{name} returned {} (not -2), cleanly", r.rax as i32 )); } } // A fault is the contract being WRONG — the shim was entered and the call did not survive. // "Could not be called" is a different thing (the injection itself did not happen) and is // deliberately NOT counted as a fault: it says nothing about the argument contract. Ok(_) => { faulted += 1; bad.push(format!("{name} FAULTED — the argument contract is wrong")); } Err(e) => bad.push(format!("{name} could not be called: {e}")), } } (probed, bailed, faulted, bad) } /// A launched, ready dedicated server the caller owns (must kill). pub(crate) struct OwnedServer { pub(crate) child: std::process::Child, pub(crate) pid: u32, } /// Launch this game's vanilla dedicated server per [`GameProfile::launch`] (no mod, no human), wait for the /// profile's readiness anchor — an alive bot PAWN for a pawn game, a live `ready_class` instance otherwise — /// then hand back the process. The single CI server-launch, shared by `produce` and `integration-test` /// (both call this). On timeout the child is killed + an error returned; on success the caller owns the /// child. pub(crate) fn launch_bots_server( prof: &GameProfile, game: &Path, build: &Path, lib: &str, wait: u64, map: &str, bots: u32, ) -> Result { let bindir = game.join("bin/linuxsteamrt64"); ensure!( bindir.join(prof.executable).exists(), "{} server executable `{}` not found at {}", prof.display_name, prof.executable, bindir.join(prof.executable).display() ); // ABSOLUTE from here on. `current_dir` below is applied in the CHILD before `exec`, so a relative // `--game-dir` would have the program path re-resolved from inside `bindir` and fail to spawn — // after the check above had just found the file, which is the worst shape for a guard to have. let bindir = bindir .canonicalize() .with_context(|| format!("resolve {}", bindir.display()))?; let exe = bindir.join(prof.executable); // Live-oracle readiness anchor (via the shared resolve_ready_anchor). A game with // a player pawn waits for an ALIVE pawn; a pawn-less game (Dota) waits for a live gamerules proxy = map // loaded + libserver ready, which is all produce's live stages (validate-live + typed netvars) need. let img = load_lib(build, lib)?; let (ready_vt, pawn_health) = resolve_ready_anchor(prof, &img)?; let logpath = std::env::temp_dir().join(format!("{}-produce.log", prof.token)); let log = std::fs::File::create(&logpath)?; let args = prof.launch.args(map, bots); let mut child = std::process::Command::new(&exe) .current_dir(&bindir) .env("LD_LIBRARY_PATH", &bindir) .env("GLIBC_TUNABLES", "glibc.rtld.execstack=2") .args(&args) // Hold stdin open (piped, never written): a dedicated Dota server quits on stdin EOF. Harmless for CS2. .stdin(std::process::Stdio::piped()) .stdout(log.try_clone()?) .stderr(log) .spawn() .with_context(|| format!("launch {}", exe.display()))?; let pid = child.id(); let ready_desc = if pawn_health.is_some() { "an alive bot pawn" } else { "the map to load" }; eprintln!( "launched vanilla {} server (pid {pid}, {bots} bots, log {}). Waiting up to {wait}s for {ready_desc}...", prof.display_name, logpath.display() ); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(wait); loop { std::thread::sleep(std::time::Duration::from_secs(3)); // A server that dies during startup is a documented failure class here (without // `sv_hibernate_when_empty 0` an empty dedicated server exits immediately, status 0). Notice it // now: otherwise the loop burns the whole timeout and then blames readiness, never mentioning // that the process is gone. if let Ok(Some(status)) = child.try_wait() { anyhow::bail!( "server pid {pid} exited with {status} before becoming ready — see {}", logpath.display() ); } // Each failure mode records WHY. Reporting only "not ready" conflates three very different causes — // memory unreadable, the library never mapped, and the instance simply not spawned yet — and leaves // the operator guessing at which. let why = match live::LiveProcess::attach(pid) { Err(e) => format!("cannot read pid {pid}'s memory: {e:#}"), Ok(live) => match live.base(lib) { None => format!("{lib} is not mapped in pid {pid}"), Some(base) => { let insts = live.find_instances(ready_vt.slot0 + base, 64); let ready = match pawn_health { Some(health) => insts .iter() .any(|&o| live.read_i32(o + health).unwrap_or(0) > 0), None => !insts.is_empty(), }; if ready { return Ok(OwnedServer { child, pid }); // ready — hand off the child } match pawn_health { Some(_) => format!("{} instance(s) found, none alive", insts.len()), None => "no instance found".to_string(), } } }, }; if std::time::Instant::now() >= deadline { let _ = child.kill(); let _ = child.wait(); anyhow::bail!( "not ready within {wait}s ({why}) — check {} (server up? map/bots loaded?)", logpath.display() ); } } } /// Validate a generated gamedata document against a LIVE server and RETURN the kept entries plus each /// entry's honest verdict and this stage's pass tally — the headless CI self-check. **Nothing is written /// here**; the caller persists what it needs (`integration-test --out`, or `produce`'s own merge). /// /// Confident-bad entries (a sig that no longer resolves or does not execute, an offset whose slot is not /// code) are dropped; ambiguous ones (base/derived offset, unknown class) are kept but reported, so /// nothing valid is silently lost. The returned [`OracleCounts`] puts this stage under the same /// `ORACLE_MIN_PASS` gate as every other one — it judges the tool's primary product, and used to be the /// only stage whose tally was printed and discarded. fn validate_live_cmd( prof: &GameProfile, pid: u32, dir: &Path, doc: &GdMap, ) -> Result<(GdMap, BTreeMap>, OracleCounts)> { let live = live::LiveProcess::attach(pid)?; // Load EVERY on-disk library that is also mapped in the live process, so a class/sig in any lib // (tier0 interfaces, networksystem, …) is validatable and an "unknown class" flag means a real // non-vtable special, not a lib we skipped. let mut lib_paths = Vec::new(); find_all_libs(dir, 8, &mut lib_paths); // Several on-disk files can share a basename (a game tree carries loader shims named exactly like the // engine libs they proxy), so first-hit-wins would validate against whichever the filesystem happened to // enumerate first. The live process is the authority on which file is actually loaded: sort its mapped // path to the front per basename, and fall back to the shallowest path when it reports none. lib_paths.sort_by_key(|p| { let mapped = p .file_name() .and_then(|s| s.to_str()) .and_then(|f| live.mapped_path(f)) .is_some_and(|m| Path::new(m) == p); (!mapped, p.components().count(), p.clone()) }); let mut images: HashMap = HashMap::new(); let mut bases: HashMap = HashMap::new(); for p in lib_paths { let Some(fname) = p.file_name().and_then(|s| s.to_str()).map(str::to_string) else { continue; }; if images.contains_key(&fname) { continue; } if let (Some(b), Ok(img)) = (live.base(&fname), CodeImage::load(&p)) { images.insert(fname.clone(), img); bases.insert(fname, b); } } ensure!( images.contains_key(prof.server_lib), "libserver.so not mapped in pid {pid}" ); let idx = build_vt_index(prof, &images); eprintln!( "indexed {} classes across {} mapped libs", idx.by_name.len(), images.len() ); let mut kept = GdMap::new(); // Per-entry validation verdict for annotate_validation: Some(true) = live-confirmed, None = kept but // UNVERIFIABLE this run (so the artifact must not claim validation), absent = dropped (=> Some(false)). let mut verdicts: BTreeMap> = BTreeMap::new(); let mut t = ValTally::default(); for (name, entry) in doc { // EVERY locator the entry carries is checked, not the first one found. An entry may hold a // signature AND a vtable offset — `model::Entry` says so, and five CS2 `core` entries do — and an // `else if` here validated only the signature while the artifact went on to stamp // `validated: true`, asserting a live check of a slot nobody read. Those slots ship into // ModSharp's `VFuncs` and Metamod's `Offsets`, where a wrong one crashes the consumer. let e = render::entry_from_value(entry); let mut outcomes: Vec = Vec::new(); if let Some(sig) = e.signature { let fname = lib_filename(prof, &sig.library); let v = validate_sig(&images, &bases, &live, &fname, Some(sig.linux.as_str())); outcomes.push(apply_sig(&mut t, name, v)); } if let Some(off) = e.offset { let class = class_of(name); let v = validate_offset(&idx, class, off, &bases, &live); outcomes.push(apply_offset(&mut t, name, off, class, v)); } // Combining is where the honesty lives. `Some(true)` requires EVERY locator to have been // confirmed; one confidently-bad locator drops the entry whatever the other says; and anything // the oracle could not check degrades the whole entry to `None` rather than letting a checked // half vouch for an unchecked one. let verdict = if outcomes.is_empty() { Some(None) // no locator at all — nothing to check, and nothing claimed } else if outcomes.iter().any(|o| matches!(o, LocatorOutcome::Drop)) { None // dropped: `annotate_validation` reads an absent verdict as Some(false) } else if outcomes .iter() .all(|o| matches!(o, LocatorOutcome::Kept(Some(true)))) { Some(Some(true)) } else if outcomes .iter() .any(|o| matches!(o, LocatorOutcome::Kept(Some(false)))) { Some(Some(false)) } else { Some(None) }; if let Some(v) = verdict { kept.insert(name.clone(), entry.clone()); verdicts.insert(name.clone(), v); } } println!("validate-live vs pid {pid} ({} entries):", doc.len()); println!( " signatures : {} live-valid ({} hooked), {} unvalidated (lib not mapped), {} dropped", t.sig_ok, t.sig_hooked, t.sig_unval, t.sig_fail ); println!( " offsets : {} live-valid ({} via subclass), {} dropped (not code), {} flagged (oob), {} flagged (not a vtable class)", t.off_live, t.off_sub, t.off_bad, t.off_oob, t.off_unknown ); for d in &t.dropped { println!(" DROP {d}"); } for f in t.flagged.iter().take(60) { println!(" FLAG {f}"); } if t.flagged.len() > 60 { println!(" … and {} more flags", t.flagged.len() - 60); } // The stage's own pass rate, returned so the release gate covers it like every other oracle stage. // This is the one stage that judges the tool's PRIMARY product — the derived signatures and vtable // offsets — and it was the one stage whose tally was printed and discarded. A derive that resolved // onto the wrong functions (stale model, mismatched --target, a changed lib set) has every entry // dropped, `annotate_validation` stamps `Some(false)` across core and high_confidence, and the release // still wrote at exit 0. `Unvalidatable`/`Unknown` are excluded rather than counted as failures: the // oracle could not judge them at all, which is what the three-valued `validated` already says. let counts = OracleCounts { checked: t.sig_ok + t.sig_fail + t.off_live + t.off_bad + t.off_oob, ok: t.sig_ok + t.off_live, faulted: t.sig_fail + t.off_bad + t.off_oob, }; Ok((kept, verdicts, counts)) } fn verify_live_cmd(prof: &GameProfile, pid: u32, dir: &Path, lib: &str) -> Result { let img = load_lib(dir, lib)?; let classes = schema::enumerate_schema(&img); // A floor, not merely non-empty: this half compares an offline read against a live one through the same // `CI_*` constants on the same bytes, so a reshape's survivors agree with themselves at ~1.0 and a // handful of classes looks like a clean run. // // It is `min_schema_classes_LIB`, because this enumerates ONE library while `produce`'s floor counts the // union across all of them — see `GameProfile::min_schema_classes_lib`. Only `server_lib` has a // calibrated count, so any other library is enumerated and reported rather than judged against a number // that does not describe it. if lib == prof.server_lib { ensure!( classes.len() >= prof.min_schema_classes_lib, "offline schema derivation found {} classes in {lib} (floor {}) — refusing to verify a schema \ whose class table collapsed", classes.len(), prof.min_schema_classes_lib ); } else { eprintln!( "NOTE: {lib} is not {}, which is the only library with a calibrated class floor — \ enumerated {} classes, collapse check SKIPPED", prof.server_lib, classes.len() ); } let live = live::LiveProcess::attach(pid)?; let base = live .base(lib) .with_context(|| format!("{lib} is not mapped in process {pid}"))?; eprintln!( "{lib} loaded at {base:#x} in pid {pid}; verifying {} schema classes...", classes.len() ); // Class-level checks across every derived class: (1) the relocated name pointer in live memory // equals our offline `.rela.dyn`-resolved pointer + slide (our reloc logic == the loader); // (2) name/size/field-count match; (3) m_pSchemaBinding is runtime-populated (we read live state). let (mut reloc_ok, mut layout_ok, mut binding_live, mut checked, mut read_err) = (0, 0, 0, 0, 0); let mut mism: Vec = Vec::new(); for c in &classes { let rci = c.class_info + base; // runtime address of the SchemaClassInfoData_t let res = (|| -> Result<()> { checked += 1; let live_name_ptr = live.read_u64(rci + schema::CI_NAME)?; if live_name_ptr == c.name_ptr + base { reloc_ok += 1; } let live_name = live.read_cstr(live_name_ptr)?; let live_size = live.read_i32(rci + schema::CI_SIZE)?; let live_fcount = live.read_u16(rci + schema::CI_FIELD_COUNT)? as usize; if live_name == c.name && live_size == c.size && live_fcount == c.fields.len() { layout_ok += 1; } else if mism.len() < 5 { mism.push(format!( "{}: live(name={live_name:?} size={live_size} fc={live_fcount}) vs offline(size={} fc={})", c.name, c.size, c.fields.len() )); } if live.read_u64(rci + schema::CI_BINDING)? != 0 { binding_live += 1; } Ok(()) })(); if res.is_err() { read_err += 1; } } println!("Runtime oracle — offline schema vs live process {pid}:"); println!(" classes checked: {checked} ({read_err} unreadable)"); println!( " relocation logic == loader: {reloc_ok}/{checked} name pointers match (offline .rela.dyn + slide == live)" ); println!(" layout (name/size/#fields): {layout_ok}/{checked} match"); println!( " m_pSchemaBinding populated: {binding_live}/{checked} (0 on disk -> proves we read LIVE runtime state)" ); if !mism.is_empty() { println!(" mismatches (sample): {mism:?}"); } // The layout agreement IS the netvars verdict: `live_schema` takes field offsets from the offline // image, and `model::Schema` has no per-field validated flag, so this is the only thing standing // between a reshaped runtime struct and a shipped netvars file. let counts = OracleCounts { checked, ok: layout_ok, faulted: read_err, }; // Spotlight: verify famous field offsets live, field-by-field (the netvars mods actually read). for &(cls, want) in prof.spotlight_fields { if let Some(c) = classes.iter().find(|c| c.name == cls) { let fields_ptr = img.read_ptr(c.class_info + schema::CI_FIELDS).unwrap_or(0) + base; println!(" {cls} fields read live from the running server:"); for &name in want { if let Some((i, f)) = c.fields.iter().enumerate().find(|(_, f)| f.name == name) { let live_off = live .read_i32( fields_ptr .wrapping_add((i as u64).wrapping_mul(schema::F_STRIDE)) .wrapping_add(schema::F_OFFSET), ) .unwrap_or(-1); let tag = if live_off == f.offset { "OK" } else { "MISMATCH" }; println!( " {name:<16} offline +{:<5} live +{live_off:<5} [{tag}]", f.offset ); } } } } Ok(counts) } /// Verdict for one vtable-offset entry validated against the live process. enum OffVerdict { Live, // in-bounds; the live runtime vtable slot points at executable code LiveViaSub(String), // valid at that slot on a subclass's vtable (the entry named a base class) NotCode, // slot target isn't executable — a wrong offset Oob, // slot exceeds the class's and its subclasses' vtables — review Unknown, // not a vtable class in any mapped lib (an engine special / member offset) } /// Verdict for one signature entry validated against the live process. enum SigVerdict { Ok, // resolves uniquely to executable live code, bytes match disk Hooked, // resolves fine but live bytes differ — a mod detoured it (sig is correct) Unvalidatable(String), // its library isn't mapped in this process — keep + flag, can't check Fail(String), // no match / ambiguous / non-executable — drop } /// Whole-binary vtable index over every mapped library: class name -> (lib, vtable vaddr, slot count) /// for the primary vtable, plus base -> direct-subclass edges (to resolve base/derived offset naming). struct VtIndex { by_name: HashMap, subs: HashMap>, } fn build_vt_index(prof: &GameProfile, images: &HashMap) -> VtIndex { // Enumerate every library's primary vtables in parallel (each is a full reloc-driven RTTI sweep), // then merge in SORTED-filename order. Sorted (not HashMap-iteration) order makes the result // deterministic: for a class exported by two libs the larger vtable wins, ties break by the // earlier filename — previously the tie winner and the subclass-list order depended on hash order. let mut names: Vec<&String> = images.keys().collect(); names.sort(); let per_lib: Vec<(&String, Vec)> = parallel_map(&names, default_threads(None), |&fname| { let cvs = rtti::enumerate_vtables(&images[fname], prof.max_vtable_slots) .into_iter() .filter(|cv| cv.offset_to_top == 0) // primary (complete-object) vtables only .collect(); (fname, cvs) }); let mut by_name: HashMap = HashMap::new(); let mut subs: HashMap> = HashMap::new(); for (fname, cvs) in &per_lib { for cv in cvs { let n = cv.slots.len(); by_name .entry(cv.name.clone()) .and_modify(|e| { if n > e.2 { *e = ((*fname).clone(), cv.vtable_va, n); } }) .or_insert(((*fname).clone(), cv.vtable_va, n)); for b in &cv.bases { subs.entry(b.name.clone()) .or_default() .push(cv.name.clone()); } } } VtIndex { by_name, subs } } /// Every `.so` under `dir` (bounded recursion) — the full library set to index. fn find_all_libs(dir: &Path, depth: usize, out: &mut Vec) { if depth == 0 { return; } let Ok(rd) = std::fs::read_dir(dir) else { return; }; for e in rd.flatten() { let p = e.path(); if p.is_dir() { find_all_libs(&p, depth - 1, out); } else if p.extension().and_then(|s| s.to_str()) == Some("so") { out.push(p); } } } /// Is the live runtime vtable slot `off` (of the vtable at file vaddr `va` in `fname`) executable? fn live_slot_is_code( va: u64, fname: &str, off: usize, bases: &HashMap, live: &live::LiveProcess, ) -> bool { let Some(&base) = bases.get(fname) else { return false; }; let ptr = live.read_u64(va + base + off as u64 * 8).unwrap_or(0); ptr != 0 && live.is_exec(ptr) } /// A method named on a base class may sit at slot `off` on a subclass's (larger) vtable. BFS the /// subclass graph for one whose vtable holds an executable slot there; return that subclass's name. fn resolve_via_subclass( idx: &VtIndex, class: &str, off: usize, bases: &HashMap, live: &live::LiveProcess, ) -> Option { let mut seen: HashSet = HashSet::new(); let mut frontier: Vec = idx.subs.get(class).cloned().unwrap_or_default(); for _ in 0..6 { let mut next = Vec::new(); for d in std::mem::take(&mut frontier) { if !seen.insert(d.clone()) { continue; } if let Some((fname, va, n)) = idx.by_name.get(&d) && off < *n && live_slot_is_code(*va, fname, off, bases, live) { return Some(d); } if let Some(s) = idx.subs.get(&d) { next.extend(s.iter().cloned()); } } if next.is_empty() { break; } frontier = next; } None } fn validate_offset( idx: &VtIndex, class: &str, off: i64, bases: &HashMap, live: &live::LiveProcess, ) -> OffVerdict { if off < 0 { return OffVerdict::NotCode; } let off = off as usize; match idx.by_name.get(class) { Some((fname, va, n)) if off < *n => { if live_slot_is_code(*va, fname, off, bases, live) { OffVerdict::Live } else { OffVerdict::NotCode } } Some(_) => match resolve_via_subclass(idx, class, off, bases, live) { Some(d) => OffVerdict::LiveViaSub(d), None => OffVerdict::Oob, }, None => OffVerdict::Unknown, } } /// A byte diff at a uniquely-resolved prologue means a mod detoured the function (evidence the sig is /// correct) → `Hooked`, kept + flagged. Only a miss / ambiguous / non-executable target is a hard fail. fn validate_sig( images: &HashMap, bases: &HashMap, live: &live::LiveProcess, fname: &str, linux: Option<&str>, ) -> SigVerdict { let (Some(img), Some(&base)) = (images.get(fname), bases.get(fname)) else { return SigVerdict::Unvalidatable(format!("lib {fname} not mapped in process")); }; let Some(pat) = linux.and_then(|l| Pattern::parse(l).ok()) else { return SigVerdict::Fail("absent/unparseable pattern".into()); }; let addr = match img.find(&pat).as_slice() { [a] => *a, [] => return SigVerdict::Fail("no live match".into()), hits => return SigVerdict::Fail(format!("{} matches (ambiguous)", hits.len())), }; let rt = addr + base; if !live.is_exec(rt) { return SigVerdict::Fail("resolved address not executable".into()); } if let Some(disk) = img.code_at(addr) { let n = disk.len().min(32); let got = live.read_bytes(rt, n); if got.len() == n && got != disk[..n] { return SigVerdict::Hooked; } } SigVerdict::Ok } /// Running tally for a validate-live pass (keeps the entry loop flat / low-complexity). #[derive(Default)] struct ValTally { sig_ok: u32, sig_hooked: u32, sig_unval: u32, sig_fail: u32, off_live: u32, off_sub: u32, off_bad: u32, off_oob: u32, off_unknown: u32, dropped: Vec, flagged: Vec, } /// What the oracle concluded about ONE locator of an entry. /// /// An entry may carry a signature AND a vtable offset — `model::Entry`'s own doc says so — and each is a /// separate claim about the running server. Judging them separately, then combining, is what stops the /// artifact stamping `validated: true` on an entry whose offset was never read. enum LocatorOutcome { /// Confidently bad: this locator does not describe the running server. Drop, /// Kept, with the honest three-valued verdict — `None` means the oracle could not check it. Kept(Option), } fn apply_sig(t: &mut ValTally, name: &str, v: SigVerdict) -> LocatorOutcome { match v { SigVerdict::Ok => { t.sig_ok += 1; LocatorOutcome::Kept(Some(true)) } SigVerdict::Hooked => { t.sig_ok += 1; t.sig_hooked += 1; t.flagged.push(format!( "{name} — sig valid; function is HOOKED live (a mod detoured it)" )); LocatorOutcome::Kept(Some(true)) // the sig IS valid; a hook is a runtime overlay, not a mismatch } SigVerdict::Unvalidatable(why) => { t.sig_unval += 1; t.flagged .push(format!("{name} — sig kept, unvalidated: {why}")); LocatorOutcome::Kept(None) // lib not mapped this run -> never claim Some(true) } SigVerdict::Fail(why) => { t.sig_fail += 1; t.dropped.push(format!("{name} — sig: {why}")); LocatorOutcome::Drop } } } fn apply_offset( t: &mut ValTally, name: &str, off: i64, class: &str, v: OffVerdict, ) -> LocatorOutcome { match v { OffVerdict::Live => { t.off_live += 1; LocatorOutcome::Kept(Some(true)) } OffVerdict::LiveViaSub(d) => { t.off_live += 1; t.off_sub += 1; t.flagged.push(format!( "{name} — offset {off} valid on subclass {d} (entry names a base class)" )); LocatorOutcome::Kept(Some(true)) // the slot IS executable on a subclass vtable — validated } OffVerdict::NotCode => { t.off_bad += 1; t.dropped .push(format!("{name} — offset {off}: slot not executable")); LocatorOutcome::Drop } OffVerdict::Oob => { t.off_oob += 1; t.flagged.push(format!( "{name} — offset {off} exceeds {class} and its subclasses' vtables" )); LocatorOutcome::Kept(Some(false)) // exceeds the vtable = likely WRONG (render drops it) } OffVerdict::Unknown => { t.off_unknown += 1; t.flagged.push(format!( "{name} — {class} is not a vtable class (engine special / member offset)" )); LocatorOutcome::Kept(None) // not a vtable class -> the oracle cannot check it } } } #[cfg(test)] mod tests { use super::*; // `DigestChange::classify` IS the CI branch decision: `skip` publishes only a manifest, `shift` marks a // toolchain jump. The thresholds are user-tunable; the calibration behind the defaults is recorded ONCE, // in `classify-change --shift-above`'s help. The cases below pin its extremes so a threshold edit has // to face them — deliberately not restated here, since two copies of a measurement is how the last one // came to be quoted long after it was superseded. fn ch(n_new: u32, common: u32) -> DigestChange { DigestChange { n_prev: n_new, n_new, common, } } #[test] fn classify_defaults_straddle_the_measured_gap() { // The extremes measured over 344 CS2 builds at ~70,300 functions each. The two INNER values are // the ones that matter: the largest ordinary patch and the smallest toolchain jump sit 4.6 // points apart, and the default has to land between them. let (skip_below, shift_above) = (0.0, 0.20); // code-identical -> skip (82 of 344 builds) assert!(matches!( ch(10_000, 10_000).classify(skip_below, shift_above), ChangeVerdict::Skip )); // the smallest real patch, 0.001% -> normal, NOT skip, at the default 0 tolerance assert!(matches!( ch(100_000, 99_999).classify(skip_below, shift_above), ChangeVerdict::Normal )); // the LARGEST ordinary patch, 17.78% -> still normal assert!(matches!( ch(10_000, 8_222).classify(skip_below, shift_above), ChangeVerdict::Normal )); // the SMALLEST toolchain jump, 22.36% -> shift assert!(matches!( ch(10_000, 7_764).classify(skip_below, shift_above), ChangeVerdict::Shift )); // and the largest, 93.8% assert!(matches!( ch(10_000, 620).classify(skip_below, shift_above), ChangeVerdict::Shift )); } #[test] fn function_digests_do_not_depend_on_eh_frame() { // The defect this pins: enumerating only `.eh_frame` FDEs samples the statically-linked runtime // tail — 8,327 of libserver's ~70,000 functions — and reports "nothing changed" about a build // whose gameplay code moved. A synthetic image has no `.eh_frame` at all, so an FDE-only // population yields NOTHING here, and `classify-change` would then answer `skip` from an empty // sample. call rel32 +0 ; ret ; xor eax,eax ; ret — the call target is a second entry. let img = CodeImage::for_test( 0x1000, &[0xE8, 0x00, 0x00, 0x00, 0x00, 0xC3, 0x31, 0xC0, 0xC3], ); assert!(img.eh_frame_functions().is_empty(), "fixture has no FDEs"); assert!( !function_digests(&img).is_empty(), "digests must come from the relocation/call-target union too, not FDEs alone" ); } #[test] fn skip_below_tolerance_absorbs_small_patches() { // raising the tolerance to 1% reclassifies the 0.08% patch as skip, and must not touch a shift assert!(matches!( ch(10_000, 9_992).classify(0.01, 0.20), ChangeVerdict::Skip )); assert!(matches!( ch(10_000, 6_600).classify(0.01, 0.20), ChangeVerdict::Shift )); } #[test] fn an_empty_new_build_is_not_a_shift() { // frac() guards n_new == 0; without that guard this would divide by zero assert!(matches!(ch(0, 0).classify(0.0, 0.20), ChangeVerdict::Skip)); } #[test] fn lcg_coin_is_not_periodic() { // Guards the LCG coin: returning low bits would make `below(2)` a period-4 sequence, so the // call-vs-netvar coin would be effectively constant across consecutive fuzz iterations. let mut r = Lcg(0x5137); let draws: Vec = (0..1000).map(|_| r.below(2)).collect(); assert!( draws .chunks(4) .filter(|c| c.len() == 4) .any(|c| c[0] != c[2] || c[1] != c[3]), "below(2) still looks 4-periodic" ); let ones = draws.iter().filter(|&&d| d == 1).count(); assert!( (400..=600).contains(&ones), "coin badly biased: {ones}/1000 ones" ); } }