//! The OFFLINE derivation engine: catalogue resolution, corpus-model distillation, gamedata folding, //! the experimental band, and monolith assembly. Everything here is a pure function of files on disk — //! **nothing in this module attaches to a running process.** Live validation, the typed schema walk and //! change classification are `produce`'s, and the split is the design rather than an accident: it is what //! lets an offline build produce a smaller honest artifact with no server anywhere in sight. //! //! The `source2rosetta` binary is a thin clap front-end over this module and `produce` (see `main.rs`); //! keeping the engine in the library lets CI and tests link and call it directly instead of shelling out. use crate::elf::CodeImage; use crate::locate::{find_file, load_lib}; use crate::par::{default_threads, parallel_map}; use crate::profile::GameProfile; use crate::sig::Pattern; use crate::taxonomy::{ FullName, class_of, clean_offset_class, guess_tier, is_dead_weight_class, is_dead_weight_name, is_serializer_plumbing, parse_ret, protobuf_message_classes, }; use crate::{abi, concmd, emit, fingerprint, model, pulse, render, rtti, schema, valvetab, xref}; use anyhow::{Context, Result, bail, ensure}; use serde::Deserialize; use serde_json::{Value, json}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; // ══════════════════════════════════════════════════════════════════════════════════════════════ // TABLE OF CONTENTS (grep `§N ·` to jump) // §1 · Catalogue model & shared vocabulary §5 · Monolith assembly // §2 · Catalogue & image loading §6 · Corpus-model distillation // §3 · Fingerprint alignment §7 · Gamedata derivation (catalogue → sigs/offsets) // §4 · Gamedata fold & experimental band §8 · Backfill (cross-build timelines) // This module is the OFFLINE engine — nothing here attaches to a running server. Companion modules: // `taxonomy` (name/dead-weight predicates), `schema` (offline reader + live type walk), `produce` // (CI orchestration + the whole LIVE verification subsystem — oracle / validate-verify / fuzzer / launch). // ══════════════════════════════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════════════════════════════ // §1 · CATALOGUE MODEL & SHARED VOCABULARY // ══════════════════════════════════════════════════════════════════════════════════════════════ /// class name -> its vtable slot fingerprints (index = slot; `None` where a slot didn't decode). /// The recurring shape behind every cross-build vtable alignment. type VtableFps = HashMap>>>; /// Candidate target addresses for one function, each tagged with how many distinct era-sigs voted /// for it (most-voted first). type VoteList = Vec<(u64, usize)>; /// Where `gamedata` pulls its cross-build reference signals from — exactly one of the two. #[derive(Clone, Copy)] pub(crate) enum CorpusSource<'a> { /// The raw corpus of build binaries (each fingerprinted on the fly). Binaries(&'a Path), /// A distilled corpus model (already parsed) — then only the target binary is read. The caller owns the /// model so it can be parsed ONCE and reused (`produce` derives through this borrow, then hands the same /// model by value to the sidecar fold — the ~571 MB Dota model is parsed a single time, not twice). Model(&'a CorpusModel), } #[derive(Deserialize)] struct Func { name: String, #[serde(default)] library: Vec, #[serde(default)] variants: Vec, } /// A catalogue variant's locator kind. `Other` (via serde) absorbs any future/unknown kind, so an /// unexpected value is skipped rather than a hard parse error. #[derive(Deserialize, PartialEq, Eq, Clone, Copy)] #[serde(rename_all = "kebab-case")] enum VariantKind { Offset, Signature, StringAnchor, VtableOffset, #[serde(other)] Other, } /// The platform a variant applies to. A locator is used for the linux derive when it is `Linux` or the /// platform-agnostic `Any`. #[derive(Deserialize, PartialEq, Eq, Clone, Copy, Default)] #[serde(rename_all = "lowercase")] enum Platform { Linux, Any, Windows, #[serde(other)] #[default] Other, } impl Platform { fn is_linux(self) -> bool { matches!(self, Platform::Linux | Platform::Any) } } #[derive(Deserialize)] struct Variant { kind: VariantKind, #[serde(default)] platform: Platform, value: String, #[serde(default)] src: String, // era tag (v1.0.N) for gamedata-sourced variants } /// Which classes get vtable-slot hops in a distilled corpus model. clap validates the choice at parse /// time (and lists it in --help), so `corpus-model` can match it exhaustively — no silent fallback. #[derive( Clone, Copy, PartialEq, Eq, Debug, Default, clap::ValueEnum, serde::Serialize, serde::Deserialize, )] #[serde(rename_all = "kebab-case")] pub enum ClassScope { /// Every real game class (the default) — enough for any modding offset to derive model-only. #[default] Clean, /// Also template / protobuf / NetworkVar junk. All, /// Only the classes the catalogue names. Catalogue, } // ══════════════════════════════════════════════════════════════════════════════════════════════ // §2 · CATALOGUE & IMAGE LOADING // ══════════════════════════════════════════════════════════════════════════════════════════════ pub(crate) fn lib_filename(prof: &GameProfile, lib: &str) -> String { match lib { "server" => prof.server_lib.into(), "engine2" => prof.engine_lib.into(), other => format!("lib{other}.so"), } } fn lib_name_from_file(fname: &str) -> String { fname .strip_prefix("lib") .unwrap_or(fname) .strip_suffix(".so") .unwrap_or(fname) .to_string() } fn candidate_libs(prof: &GameProfile, f: &Func) -> Vec { let named: Vec<&String> = f.library.iter().filter(|l| *l != "unknown").collect(); if named.is_empty() { vec![prof.server_lib.into(), prof.engine_lib.into()] } else { named.iter().map(|l| lib_filename(prof, l)).collect() } } /// Why no pass could produce a locator for `f` — the DETAIL that ships beside its `unresolved` entry. /// /// Worth distinguishing rather than saying "not found", because the three cases are different claims and /// only one of them is about this build. Most of what lands here was never a function: the catalogue is /// harvested from other people's dumps, and a dumper's own JSON keys (`build_number`, `dwEntityList`, /// `attack`) arrive looking exactly like names. Reporting those as functions this build failed to locate /// would be its own kind of lie — the artifact would assert that CS2 has a function called `jump`. fn unresolvable_because(f: &Func) -> &'static str { let kinds: Vec = f.variants.iter().map(|v| v.kind).collect(); if !kinds.is_empty() && kinds.iter().all(|k| *k == VariantKind::Offset) { // A raw member offset, not a vtable slot — the kind `ContributionKind` deliberately excludes. return "catalogue carries only `offset` variants — a raw member offset, which this tool does \ not derive (it derives signatures and VTABLE slots). Often not a function at all."; } if !linux_sigs(f).is_empty() && f.library.iter().all(|l| l == "unknown") { return "signature variants name no library, so there is no image to scan — the harvested source \ did not record one"; } if linux_sigs(f).is_empty() && string_anchors(f).is_empty() { return "no linux signature, no string anchor and no vtable offset — nothing this build knows \ how to look for"; } "no locator produced by any pass — the signature did not resolve uniquely and no string anchor \ referenced exactly one function in the target" } fn linux_sigs(f: &Func) -> Vec<&Variant> { f.variants .iter() .filter(|v| v.kind == VariantKind::Signature && v.platform.is_linux()) .collect() } /// String-anchor locators for `f`: distinctive strings the function references. Stable across builds /// even when byte sigs drift, so `gamedata` re-resolves them per build via the xref index. fn string_anchors(f: &Func) -> Vec<&str> { f.variants .iter() .filter(|v| v.kind == VariantKind::StringAnchor && v.platform.is_linux()) .map(|v| v.value.as_str()) .collect() } pub(crate) fn label_of(dir: &Path) -> String { dir.file_name() .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_else(|| dir.display().to_string()) } /// Load every library the catalogue references (usually just libserver/libengine2) from a build, plus the /// names that were PRESENT and would not parse. /// /// The two failures are not the same fact and must not read the same. A library the catalogue names but /// this tree does not have is ordinary — `library` is community-sourced and routinely wrong, and /// `seed-cs2.json` names `client`, `hammer` and `undefined` among others, none of which exist in a /// dedicated-server tree. A library that IS there and does not parse is a broken input, and every catalogue /// entry naming it then resolves nowhere and ships as `SigDrifted` — "no unique/recovered signature in /// target" — asserting that a signature drifted inside a file nobody opened. fn preload_images( prof: &GameProfile, funcs: &[Func], dir: &Path, ) -> (HashMap, Vec) { let mut wanted: BTreeSet = BTreeSet::new(); for f in funcs { if !linux_sigs(f).is_empty() { wanted.extend(candidate_libs(prof, f)); } } let mut images = HashMap::new(); let mut unreadable = Vec::new(); for fname in &wanted { let Some(p) = find_file(dir, fname, 8) else { continue; // absent: normal, see above }; match CodeImage::load(&p) { Ok(img) => { images.insert(fname.clone(), img); } Err(e) => unreadable.push(format!("{fname} ({e:#})")), } } (images, unreadable) } /// Every distinct address a UNIQUE-matching, de-duplicated era-sig of `f` resolves to in `img`, each mapped /// to how many distinct era-sigs voted for it. The shared scan behind `locate_addr` / `locate_candidates` / /// `resolve_unique` — they differ only in how they aggregate this map (unique-key / vote-ranked / cross-image /// union). A sig with a non-unique match (0 or >1 hits) contributes nothing; the seen-set dedups equal sigs. fn scan_sig_hits(f: &Func, img: &CodeImage) -> BTreeMap { let mut votes: BTreeMap = BTreeMap::new(); let mut seen_sig = HashSet::new(); for v in linux_sigs(f) { if !seen_sig.insert(v.value.as_str()) { continue; } if let Ok(pat) = Pattern::parse(&v.value) && let [addr] = img.find(&pat).as_slice() { *votes.entry(*addr).or_default() += 1; } } votes } /// Two things the binary states about an address, kept per image so a name can be checked against both. /// /// Built once per image set and shared read-only across the resolution threads. Both halves are OFFLINE — /// the VScript registry constant-propagates out of its initialiser and the SchemaSystem states each class's /// instance size in static data — so this check runs identically in an offline derive, a live one, and the /// distill. struct LibIdentity { /// Implementation address -> the C++ name Valve's VScript registry registers there. Ground truth: it is /// the binary naming its own function, not an inference about it. vscript_at: HashMap, /// Schema class -> instance size in bytes, as the class states it. class_size: HashMap, } /// The identity evidence for every preloaded image, keyed by SHORT library name (`server`) — never the /// file name it was loaded from. See [`Identity::contradiction`] for why that distinction is load-bearing. pub(crate) struct Identity(HashMap); impl Identity { fn of(images: &HashMap) -> Self { Identity( images .iter() .map(|(fname, img)| { let vscript_at = crate::vscript::vscript_functions(img) .into_iter() .filter_map(|v| match v.imp { // Only a plain address identifies a function here; a virtual `Slot` names a // vtable index, which locates nothing without the class. Some(crate::vscript::Impl::Addr(a)) if a != 0 => Some((a, v.cpp_name)), _ => None, }) .collect(); let class_size = schema::enumerate_schema(img) .into_iter() .filter(|c| c.size > 0) .map(|c| (c.name, c.size as u64)) .collect(); ( lib_name_from_file(fname), LibIdentity { vscript_at, class_size, }, ) }) .collect(), ) } /// Why `addr` CONTRADICTS `name`, or `None` if it does not. /// /// **A conjunction of two independent contradictions, and it needs both.** Either alone is too noisy to /// reject on, which is the whole reason this is shaped as an AND: /// /// - *Valve's registry names it something else.* Alone this rejects real aliases — a dozen CS2 bindings /// are bound STRAIGHT to the native method rather than through a script wrapper, so `SetAbsOrigin` and /// `CBaseEntity::SetAbsOrigin` are legitimately one address, as is `ScriptSetSize` / /// `CBaseModelEntity::SetCollisionBounds` whose names do not even resemble each other. /// - *The code says it is a different class.* Alone this rejects working locators whose NAME merely /// carries the wrong class prefix — measured: four `CPathMover::` entries that are really `CFuncMover` /// setters, and two `CBasePlayerController::` entries that are really `CCSPlayerController`. Those /// locate correctly; only their qualifier is wrong, and dropping them would lose real call sites. /// /// Together they fire on the case where Valve names the address one thing and the machine code agrees it /// is not the class the catalogue claims. Measured across 3,988 CS2 entries: **exactly one hit**, and it /// was a genuine defect that had shipped in a release (`CBaseEntity::DispatchTraceAttack`, in fact /// `CLogicRelay::Trigger`). n=1, so this rejects one entry rather than aborting a release. fn contradiction(&self, lib: &str, img: &CodeImage, addr: u64, name: &str) -> Option { // `Identity::of` gives every loaded image an entry, and both call sites key from a file in that // same map, so a miss here does NOT mean "a library with no evidence" — it means the caller holds // the wrong FORM of the key, and the `?` this used to be then disabled the entire check without a // word. That is precisely how it sat dead on the fold path. // // NOT a `debug_assert!`: that compiles out in release, and the shipped binary is a release build, // so the guard would have gone on silently passing in the one configuration that matters. let Some(lib) = self.0.get(lib) else { panic!("Identity is keyed by SHORT library name (`server`), got `{lib}`"); }; let (class, method) = name.split_once("::")?; let registered = lib.vscript_at.get(&addr)?; if names_correspond(registered, method) { return None; // Valve's name for this address agrees — a direct binding, not a contradiction } let size = *lib.class_size.get(class)?; let reach = abi::this_reach(img, addr)?; (reach >= size).then(|| { format!( "Valve's VScript registry registers {addr:#x} as `{registered}`, and the code reaches \ `this+{reach}` on a `{class}` of {size} bytes — the address is not this function" ) }) } } /// Whether a VScript-registered C++ name and a catalogue method name are the same function under two /// spellings. Valve prefixes script-bound wrappers inconsistently (`ScriptSetAbsAngles`, `Script_TakeDamage`, /// bare `SetAbsOrigin`), so the prefix is stripped before comparing, and containment either way is accepted /// because the two vocabularies abbreviate differently (`GetEHandle` / `GetRefEHandle`). fn names_correspond(registered: &str, method: &str) -> bool { let norm = |s: &str| { s.trim_start_matches("Script_") .trim_start_matches("Script") .to_ascii_lowercase() }; let (a, b) = (norm(registered), norm(method)); !a.is_empty() && !b.is_empty() && (a.contains(&b) || b.contains(&a)) } /// The unique address of `f` in the preloaded images, if exactly one variant resolves cleanly. fn locate_addr<'a>( prof: &GameProfile, f: &Func, images: &'a HashMap, ) -> Option<(&'a CodeImage, u64, String)> { locate_addr_ident(prof, f, images, None) } /// `locate_addr`, refusing an address the binary itself contradicts (see [`Identity::contradiction`]). /// Passing `None` skips the check, which is what the callers that have no image set to build it from do. fn locate_addr_ident<'a>( prof: &GameProfile, f: &Func, images: &'a HashMap, ident: Option<&Identity>, ) -> Option<(&'a CodeImage, u64, String)> { let (img, fname) = candidate_libs(prof, f) .into_iter() .find_map(|fname| images.get(&fname).map(|img| (img, fname)))?; let hits = scan_sig_hits(f, img); let [addr] = hits.keys().copied().collect::>()[..] else { return None; }; // Derived ONCE and used for both the identity lookup and the return: the two want the same value, and // spelling it twice is how they came to disagree — the lookup was passed the FILE name against a map // keyed by the short one, so it missed on every call and the guard below never rejected anything. let lib = lib_name_from_file(&fname); if let Some(id) = ident && id.contradiction(&lib, img, addr, &f.name).is_some() { return None; } Some((img, addr, lib)) } /// Every distinct target address a unique-matching era-sig lands on, ranked by how many distinct /// era-sigs voted for it (most first). Unlike `locate_addr` this keeps ALL candidates so a caller /// can disambiguate a stale sig's coincidental hit from the real function by fingerprint. fn locate_candidates<'a>( prof: &GameProfile, f: &Func, images: &'a HashMap, ) -> Option<(&'a CodeImage, String, VoteList)> { let (img, fname) = candidate_libs(prof, f) .into_iter() .find_map(|fname| images.get(&fname).map(|img| (img, fname)))?; let hits = scan_sig_hits(f, img); if hits.is_empty() { return None; } let mut cand: Vec<(u64, usize)> = hits.into_iter().collect(); cand.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0))); // most-voted first, addr as tiebreak Some((img, lib_name_from_file(&fname), cand)) } fn load_catalogue(path: &Path) -> Result> { let text = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; let funcs: Vec = serde_json::from_str(&text).context("parse catalogue json")?; // A name may appear ONCE. The catalogue is keyed by name everywhere downstream — `gd.entries`, the // flag lists, the completeness sweep — but the distill walks this Vec, so a repeat pushes that name's // slot observation twice and gives one function two votes in its own timeline. `gamedata` already // merges same-named contributions into one `Func`'s variants for exactly this reason; a duplicate // reaching here means the catalogue itself is malformed, which is an intake error rather than // something to silently average out. Same posture as `is_iso_date` and the closed `ContributionKind`. let mut seen: HashSet<&str> = HashSet::with_capacity(funcs.len()); let dups: Vec<&str> = funcs .iter() .map(|f| f.name.as_str()) .filter(|n| !seen.insert(n)) .collect(); ensure!( dups.is_empty(), "catalogue {} carries {} duplicate name(s): {}. Every consumer keys on the name, and the distill \ walks the list — a repeat gives one function two votes in its own slot timeline.", path.display(), dups.len(), dups.join(", ") ); Ok(funcs) } /// The two locator kinds a human may contribute. DELIBERATELY not `VariantKind`: that one carries a /// `#[serde(other)]` catch-all, which is right for the machine-generated catalogue (forward-compatible with /// kinds this build doesn't know) and WRONG for a hand-authored file — `"vtable_offset"` with an underscore, /// or the real-but-underivable `"offset"`, would parse cleanly, be counted as merged, and then match nothing /// downstream, producing no output and no diagnostic. A closed enum turns that typo into a parse error /// naming the file and entry. #[derive(Deserialize, Clone, Copy)] #[serde(rename_all = "kebab-case")] enum ContributionKind { Signature, VtableOffset, } impl From for VariantKind { fn from(k: ContributionKind) -> Self { match k { ContributionKind::Signature => VariantKind::Signature, ContributionKind::VtableOffset => VariantKind::VtableOffset, } } } /// One human-contributed function locator, from `mappings/contributions//*.json`: a name + a /// DATED signature or vtable-offset locator. Merged into the catalogue at DERIVE time — the unified /// contribution/PR path. The `date` is the observation's build; the derive chains/verifies it exactly like /// any dated catalogue variant. #[derive(Deserialize)] struct Contribution { name: String, kind: ContributionKind, value: String, date: String, // observation build date, YYYY-MM-DD (drives offset chaining / backfill) } /// `YYYY-MM-DD`, the shape the dated-variant machinery requires. Validated at INTAKE so a malformed date is /// reported against its file and entry, rather than silently failing the `src.len() >= 10 && src[4] == b'-'` /// test deep inside `vtable_offset_timelines`, where it just looks like a locator that didn't resolve. fn is_iso_date(s: &str) -> bool { let b = s.as_bytes(); b.len() == 10 && b[4] == b'-' && b[7] == b'-' && b.iter() .enumerate() .all(|(i, c)| matches!(i, 4 | 7) || c.is_ascii_digit()) } /// Human contributions for the active game as catalogue `Func`s (one dated variant each), read from /// `/contributions//*.json` in sorted order. Empty when the dir is absent. /// Merged by the DERIVE only — never into the historical corpus model. fn load_contributions(prof: &GameProfile, catalogue: &Path) -> Vec { let Some(dir) = catalogue .parent() .map(|p| p.join("contributions").join(prof.game_key)) .filter(|d| d.is_dir()) else { return Vec::new(); }; let mut files: Vec = std::fs::read_dir(&dir) .into_iter() .flatten() .flatten() .map(|e| e.path()) .filter(|p| p.extension().is_some_and(|x| x == "json")) .collect(); files.sort(); // deterministic merge order let mut funcs = Vec::new(); for f in &files { let text = match std::fs::read_to_string(f) { Ok(t) => t, Err(e) => { eprintln!(" contribution {}: {e} — skipped", f.display()); continue; } }; // Parse entry-by-entry via `Vec` so ONE malformed entry costs only itself, not the whole // file — CONTRIBUTING.md promises "a malformed entry is skipped with a warning". let rows: Vec = match serde_json::from_str(&text) { Ok(r) => r, Err(e) => { eprintln!( " contribution {}: not a JSON array ({e}) — file skipped", f.display() ); continue; } }; for (i, row) in rows.into_iter().enumerate() { let c: Contribution = match serde_json::from_value(row) { Ok(c) => c, Err(e) => { eprintln!(" contribution {}[{i}]: {e} — entry skipped", f.display()); continue; } }; if !is_iso_date(&c.date) { eprintln!( " contribution {}[{i}] ({}): date {:?} is not YYYY-MM-DD — entry skipped", f.display(), c.name, c.date ); continue; } let variant = Variant { kind: c.kind.into(), platform: Platform::Linux, value: c.value, src: c.date, }; // Merge onto an existing contributed Func rather than appending a duplicate. A second `Func` // of the same name would never let the contribution corroborate the catalogue's own era-sigs // (`scan_sig_hits` votes across the variants of ONE `Func`), and would let one name be counted // twice by the emit tallies. match funcs.iter_mut().find(|g: &&mut Func| g.name == c.name) { Some(g) => g.variants.push(variant), None => funcs.push(Func { name: c.name, library: Vec::new(), variants: vec![variant], }), } } } if !funcs.is_empty() { eprintln!( " merged {} contribution(s) from {}", funcs.len(), dir.display() ); } funcs } /// Immediate subdirectories of `corpus` that contain a libserver.so (i.e. builds), sorted. pub(crate) fn find_builds(prof: &GameProfile, corpus: &Path) -> Result> { let mut builds: Vec = std::fs::read_dir(corpus) .with_context(|| format!("read {}", corpus.display()))? .flatten() .map(|e| e.path()) .filter(|p| p.is_dir() && find_file(p, prof.server_lib, 8).is_some()) .collect(); builds.sort(); ensure!( !builds.is_empty(), "no build subdirs containing libserver.so under {}", corpus.display() ); Ok(builds) } /// Load a build's server/engine libraries once (reused across many class lookups). fn load_build_images(prof: &GameProfile, dir: &Path) -> Vec { load_build_images_counted(prof, dir).0 } /// As [`load_build_images`], plus the libs that were absent or unparseable. /// /// A build is admitted to the corpus on file EXISTENCE alone, so a corrupt-but-present library still /// counts as a build; it then contributes empty fingerprints, which the distill folds in as slot-count 0 /// with an EMPTY hop for every class — and an empty hop is an unconditional chain break, severing every /// anchor older than that build. That is worth a line of output rather than a silent `filter_map`. fn load_build_images_counted(prof: &GameProfile, dir: &Path) -> (Vec, Vec) { let mut imgs = Vec::new(); let mut failed = Vec::new(); for f in prof.libs { // Absent is normal — a profile's lib list is a superset across builds — so only an // unparseable file counts as a failure. if let Some(p) = find_file(dir, f, 8) { match CodeImage::load(&p) { Ok(i) => imgs.push(i), Err(_) => failed.push((*f).to_string()), } } } (imgs, failed) } // ══════════════════════════════════════════════════════════════════════════════════════════════ // §3 · FINGERPRINT ALIGNMENT // ══════════════════════════════════════════════════════════════════════════════════════════════ fn l1(a: &[u32], b: &[u32]) -> u64 { a.iter().zip(b).map(|(&x, &y)| x.abs_diff(y) as u64).sum() } /// L1 distance, but stops as soon as it passes `cap` (then returns `cap + 1`). `slot_score` only needs /// to know which of `{0, 1..=4, >4}` the distance lands in, and a Needleman–Wunsch fill scores mostly /// unrelated slot pairs whose distance blows past the cap in the first few of 63 dims — so the early /// exit turns the reduction from 63 adds into 1–3 on the hot path. Exact for the buckets `slot_score` uses. fn l1_capped(a: &[u32], b: &[u32], cap: u64) -> u64 { let mut sum = 0u64; for (&x, &y) in a.iter().zip(b) { sum += x.abs_diff(y) as u64; if sum > cap { return cap + 1; } } sum } /// Score two slot fingerprints for alignment: identical → strong match, near → weak, else /// mismatch. Undecodable slots never match. fn slot_score(a: &Option>, b: &Option>) -> i32 { match (a, b) { (Some(x), Some(y)) => match l1_capped(x, y, 4) { 0 => 3, 1..=4 => 1, _ => -1, }, _ => -1, } } /// Order-preserving (Needleman-Wunsch) alignment of two slot-fingerprint sequences, scored by /// fingerprint similarity. Returns from_idx -> Some(to_idx) on diagonal (matched-position) steps, /// None where the from-slot was deleted. Order preservation disambiguates the many near-identical /// trivial getters a greedy nearest-neighbour cannot. fn nw_align(from: &[Option>], to: &[Option>]) -> Vec> { let (n, m) = (from.len(), to.len()); // Identity fast-path: for equal sequences the all-diagonal path is the unique optimum (a match, incl. // None-None, scores > a delete+insert gap pair at every prefix), so the traceback walks the full diagonal // and returns `[Some(0)..Some(n-1)]`. This is BYTE-IDENTICAL to the full DP result — and on a code-distinct // corpus the vast majority of a class's adjacent-build vtables are unchanged, so this skips the O(n²) fill. if from == to { return (0..n).map(Some).collect(); } const GAP: i32 = -1; // Flat (n+1)×(m+1) DP matrix in ONE allocation rather than n+1 heap Vecs — this is a per-call hot path // (millions of calls per distill). let w = m + 1; let mut dp = vec![0i32; (n + 1) * w]; for i in 1..=n { dp[i * w] = GAP * i as i32; } for j in 1..=m { dp[j] = GAP * j as i32; } for i in 1..=n { for j in 1..=m { let s = slot_score(&from[i - 1], &to[j - 1]); dp[i * w + j] = (dp[(i - 1) * w + (j - 1)] + s) .max(dp[(i - 1) * w + j] + GAP) .max(dp[i * w + (j - 1)] + GAP); } } let mut map = vec![None; n]; let (mut i, mut j) = (n, m); while i > 0 && j > 0 { let s = slot_score(&from[i - 1], &to[j - 1]); if dp[i * w + j] == dp[(i - 1) * w + (j - 1)] + s { map[i - 1] = Some(j - 1); // diagonal: positions correspond (match or substitution) i -= 1; j -= 1; } else if dp[i * w + j] == dp[(i - 1) * w + j] + GAP { i -= 1; } else { j -= 1; } } map } // ══════════════════════════════════════════════════════════════════════════════════════════════ // §4 · GAMEDATA FOLD & EXPERIMENTAL BAND (dead-weight/naming predicates live in `taxonomy`) // ══════════════════════════════════════════════════════════════════════════════════════════════ /// One verified promotable (T3) name from the name-extrapolation harvest. #[derive(Deserialize)] struct PromoName { addr: String, name: String, #[serde(default)] confidence: String, #[serde(default)] rationale: String, #[serde(default)] corroboration: String, #[serde(default)] self_named: bool, #[serde(default)] promote: bool, } /// One anonymous-function context bundle from the `--candidates` input (its membership = the dead-weight filter). #[derive(Deserialize)] struct CandCtx { addr: String, #[serde(default)] class: Option, #[serde(default)] slot: Option, #[serde(default)] abi: Option, } #[derive(Deserialize, Default)] struct CandDoc { candidates: Vec, } /// A function's ABI return class — typed so the by-value guard can't be silently broken by a `"byval"` string /// typo. A struct returned BY VALUE hides an output-buffer pointer in RDI (RVO), so a naive `this`-call would /// make the callee WRITE its return into the object = memory corruption; this flag keeps such functions out of /// the blind-call path. Parsed from the candidate's `ret=` abi token; `as_str` round-trips to that token, so /// the provenance `ret_class` string is byte-unchanged. Distinct from `abi::RetClass` (the DECODED 5-way /// return class fed to the ABI-drift consensus): this one is parsed from the naming-harvest provenance token, /// carries arbitrary type strings the fixed enum can't hold, and exists ONLY for the by-value safety flag. enum ByValClass { ByValue, Other(String), } impl ByValClass { fn from_abi(abi: Option<&str>) -> ByValClass { match abi.and_then(parse_ret) { Some("byval") => ByValClass::ByValue, Some(s) => ByValClass::Other(s.to_string()), None => ByValClass::Other("?".to_string()), } } fn is_by_value(&self) -> bool { matches!(self, ByValClass::ByValue) } fn as_str(&self) -> &str { match self { ByValClass::ByValue => "byval", ByValClass::Other(s) => s, } } } /// Does this promoted entry route to a vtable OFFSET (a clean-class method at a known slot whose AI-guessed /// class matches its RTTI class) rather than a byte signature? Decided identically at the sig prefilter and /// the emit loop. fn is_offset_locator(prof: &GameProfile, pr: &PromoName, c: &CandCtx) -> bool { let ai_class = pr.name.rsplit_once("::").map(|(cl, _)| cl); c.slot.is_some() && c.class .as_deref() .is_some_and(|cls| clean_offset_class(prof, cls)) && ai_class == c.class.as_deref() } /// Parse a `0x`-prefixed hex address string (the harvest/candidate `addr` field) to a `u64`. `None` on a /// missing prefix or non-hex digits. fn parse_hex_addr(s: &str) -> Option { u64::from_str_radix(s.strip_prefix("0x")?, 16).ok() } /// A ground-truth vtable-OFFSET entry for a lib OTHER than the primary — `name` is `Class::Method` (the /// class is read from the name, exactly like the primary fold's offset entries), `slot` is its vtable /// index. Keyed by lib in the `--extra-offsets` file. The source is symbol transfer (macOS ground truth), /// so these bypass the candidate/dead-weight gate — they are verified names, not guesses. #[derive(Deserialize)] struct ExtraOffset { name: String, class: String, slot: i64, /// Provenance overrides — absent = the macOS ground-truth shape (`corroborated`/`macos-groundtruth`), /// so a plain `{name,class,slot}` stays macOS. AI-promotable virtual methods set their own tier /// (`self-named`/`dict-exact`) + source so the monolith records where each name came from. #[serde(default)] tier: Option, #[serde(default)] source: Option, #[serde(default)] confidence: Option, #[serde(default)] self_named: bool, } /// A multilib NON-virtual name to fold as a byte SIGNATURE — `addr` is its offset in lib `elib` (the /// key). `make_sig` runs on that lib's own image. Used for AI-promotable names that aren't clean vtable /// methods, keyed by lib in the `--extra-sigs` file. #[derive(Deserialize)] struct ExtraSig { name: String, addr: String, #[serde(default)] tier: Option, #[serde(default)] source: Option, #[serde(default)] confidence: Option, #[serde(default)] self_named: bool, } /// Lift an extra-fold's data-string tier to the typed enum, erroring loudly on an unrecognised id. fn tier_from_id(t: &str) -> Result { model::Tier::from_id(t).with_context(|| format!("unknown provenance tier {t:?}")) } /// Fold the multilib ground-truth vtable-OFFSET entries (macOS symbol transfer) for the non-primary server /// libs as high_confidence offsets: the class is read from the name (so the modder resolves the right lib's /// vtable) and the candidate/dead-weight gate is bypassed — these are verified ground truth, not guesses. A /// name already owned by the promoted fold (`t3`) or the guaranteed core is left untouched. Returns the count. fn fold_extra_offsets( path: &Path, core_gd: &GdMap, t3: &mut BTreeMap, prov: &mut BTreeMap, ) -> Result { let extra: BTreeMap> = serde_json::from_str(&std::fs::read_to_string(path)?) .context("parse extra-offsets json")?; let mut n = 0u32; for (elib, entries) in &extra { for e in entries { if t3.contains_key(&e.name) || core_gd.contains_key(&e.name) { continue; // a higher-precedence source already owns this name } t3.insert(e.name.clone(), model::Entry::offset(e.slot)); let source = e.source.as_deref().unwrap_or("macos-groundtruth"); prov.insert( e.name.clone(), model::Provenance { addr: Some(String::new()), confidence: Some(e.confidence.as_deref().unwrap_or("high").to_string()), self_named: Some(e.self_named), // NOT stated, because nothing here measured them. `by_value: false` is a SAFETY // claim — "returns nothing by value, so a naive call is safe" — and this fold has // only a name and a vtable slot, no address to measure at. The entry's `abi`, which // `offset_abi_shapes` fills by resolving the slot, is the one statement about this // function's return that rests on evidence. by_value: None, ret_class: None, source: Some(source.to_string()), rtti_class: Some(e.class.clone()), rationale: Some(format!("multilib vtable offset ({elib}, {source})")), ..model::Provenance::with_tier(tier_from_id( e.tier.as_deref().unwrap_or("corroborated"), )?) }, ); n += 1; } } eprintln!( " +{n} multilib ground-truth offset entries folded from {}", path.display() ); Ok(n) } /// Fold the multilib NON-virtual names as byte SIGNATURES via `make_sig` on each name's OWN lib image (loaded /// on demand, cached) — the AI-promotable names that aren't clean vtable methods. Same precedence/gate rule as /// [`fold_extra_offsets`]; a name whose sig can't be made (bad addr / no unique pattern) is skipped. Count. fn fold_extra_sigs( build: &Path, path: &Path, sig_cap: usize, core_gd: &GdMap, t3: &mut BTreeMap, prov: &mut BTreeMap, ) -> Result { let extra: BTreeMap> = serde_json::from_str(&std::fs::read_to_string(path)?).context("parse extra-sigs json")?; let mut imgs: HashMap = HashMap::new(); let mut n = 0u32; // Losses here are the norm, not the exception (~30% of this input typically doesn't fold), and a // success-only tally cannot distinguish "folded what it could" from "skipped an entire library". let (mut skipped_libs, mut bad_addr, mut unmakeable, mut overlaps) = (Vec::new(), 0u32, 0u32, 0u32); for (elib, entries) in &extra { if !imgs.contains_key(elib) { match load_lib(build, elib) { Ok(i) => { imgs.insert(elib.clone(), i); } Err(_) => { // lib not present in this build — its whole name list is skipped skipped_libs.push(format!("{elib} ({})", entries.len())); continue; } } } let eimg = &imgs[elib]; let elib_name = lib_name_from_file(elib); for e in entries { if t3.contains_key(&e.name) || core_gd.contains_key(&e.name) { overlaps += 1; continue; } let Some(addr) = parse_hex_addr(&e.addr) else { bad_addr += 1; continue; }; let Some(sig) = emit::make_sig(eimg, addr, sig_cap) else { unmakeable += 1; continue; }; let shape = abi::abi_shape(eimg, addr); t3.insert( e.name.clone(), model::Entry::signature(elib_name.clone(), sig), ); prov.insert( e.name.clone(), model::Provenance { addr: Some(e.addr.clone()), confidence: Some(e.confidence.as_deref().unwrap_or("medium").to_string()), self_named: Some(e.self_named), // MEASURED at the address this entry resolves to, not assumed. `by_value` says a // naive call is safe; asserting `false` for every folded name would vouch for a // by-value returner, which writes through a hidden pointer the caller never passed. // Where the function does not decode, both stay `None` — an absent claim. // THREE-way, not two: a decoded function whose return class is `Unknown` has not // been shown safe either. `false` is a positive claim — "not a by-value returner, so // a naive call will not write through a hidden pointer" — and it is only earned by a // return class the measurement actually settled. by_value: shape.and_then(|sh| match sh.ret_class { abi::RetClass::Unknown => None, c => Some(c == abi::RetClass::ByValue), }), ret_class: shape.map(|sh| sh.ret_class.describe().to_string()), source: Some( e.source .as_deref() .unwrap_or("source2rosetta-nameext") .to_string(), ), rationale: Some(format!("multilib name sig ({elib})")), ..model::Provenance::with_tier(tier_from_id( e.tier.as_deref().unwrap_or("contextual"), )?) }, ); n += 1; } } eprintln!(" +{n} multilib name sigs folded from {}", path.display()); if !skipped_libs.is_empty() || bad_addr > 0 || unmakeable > 0 || overlaps > 0 { eprintln!( " skipped: {overlaps} already-known, {unmakeable} unmakeable-sig, {bad_addr} bad-addr{}", if skipped_libs.is_empty() { String::new() } else { format!(", libs absent from this build: {}", skipped_libs.join(", ")) } ); } Ok(n) } // The provenance `source` ids. Declared HERE because this is where they are stamped, and `pub(crate)` // because `prototypes` reads them back to decide what a name's evidence implies — it used to re-declare // its own literals "kept in step" by a comment, which is an invariant nothing checks. /// The provenance `source` id an entity-IO datadesc name ships under. pub(crate) const VALVE_DATADESC: &str = "valve-datadesc"; /// The provenance `source` id prefix a console-command handler ships under; the callback FORM is /// appended after a colon (`valve-concommand:direct`). pub(crate) const VALVE_CONCOMMAND: &str = "valve-concommand"; /// Provenance id for a name the SCRIPT VM registry states. Same class of evidence as the datadesc and /// console-command tables: the binary names the function and gives its address in one initialiser. pub(crate) const VALVE_VSCRIPT: &str = "valve-vscript"; /// A console-command handler's gamedata name. The command name is Valve's own string; the prefix marks /// what the entry IS — the handler bound to console command `X` — rather than claiming a C++ symbol we /// do not know. `ent_fire`'s real method name is nowhere in the binary; `ConCommand::ent_fire` is. fn concmd_key(name: &str) -> String { format!("ConCommand::{name}") } /// Integer arguments in the entity-IO input prototype every datadesc handler shares — /// `void(CEntityInstance*, InputData_t&)`: two pointers, no float arguments, nothing on the stack. /// /// Hundreds of independent functions declared to one fixed prototype is a free correctness oracle for /// the backward-liveness ABI reader, so the fold MEASURES them rather than assuming. What it checks is /// the direction that matters: an OVER-count. The footprint is a documented lower bound — a handler /// that ignores its `InputData_t&` genuinely reads one register, and a forwarding thunk reads none — so /// measuring fewer arguments than declared is correct behaviour, while measuring MORE (or any float, or /// a stack argument, or a by-value return) means the reader invented a parameter that isn't there, and /// a consumer calling through it would load registers the function never reads. /// /// Reported, not gated: a future build could legitimately introduce a handler this doesn't cover, and a /// derivation should not fail on a diagnostic. const IO_HANDLER_INT_ARGS: u8 = 2; /// Does a measured shape stay within the entity-IO handler prototype (i.e. no over-count)? fn within_io_prototype(s: &abi::AbiShape) -> bool { s.int_args <= IO_HANDLER_INT_ARGS && s.float_args == 0 && !s.stack_args && s.ret_class != abi::RetClass::ByValue } /// Integer arguments in a console-command callback: `void(const CCommandContext&, const CCommand&)`, /// plus a receiver for the two forms that dispatch through an object. /// /// The same free oracle the entity-IO prototype gives, from a second direction — and this one is /// TWO-SIDED, which the entity-IO check is not. The form comes from the callback-type argument at the /// registration site; the argument count comes from a backward-liveness pass over a completely /// different function body, and neither reader knows the other exists. So a `direct` registration whose /// handler reads three integer registers would mean one of the two is wrong — either the wrong function /// was picked up, or the shape reader over-counted. Measured on CS2: **direct 699/699 within two, /// interface and member 85/85 within three, zero float arguments and zero by-value returns anywhere.** fn concmd_int_args(form: concmd::CallbackForm) -> u8 { match form { concmd::CallbackForm::Direct => 2, concmd::CallbackForm::Interface | concmd::CallbackForm::Member => 3, } } /// `PulseValueType_t::PVAL_EHANDLE`. Checkable rather than assumed: the enum is schema-registered, so /// the `schema.enums` section of the shipped artifact states the same value. const PVAL_EHANDLE: i32 = 13; /// `mappings/ehandle-classes.json` — Valve's own naming for the entity class behind each handle /// parameter, keyed by binding and parameter NAME. See `archive/valve-tables/extract-ehandle-classes.py` /// for why it is keyed that way and not by address. #[derive(serde::Deserialize)] struct EhandleClasses { classes: BTreeMap>, } /// Name the entity class behind every `PVAL_EHANDLE` parameter. /// /// The initializer states a parameter's `PulseValueType_t` and, for a handle, stops there — the engine /// resolves the entity class at runtime through a registry. But it also stores the address of the /// TYPE'S DESTRUCTOR beside each parameter, and that address is per concrete type: two parameters carry /// the same one exactly when they are the same handle type. So the binary supplies the GROUPING and /// Valve's published metadata supplies the NAME, and neither alone is enough. /// /// Grouped per LIBRARY because the token is an address: tokens from `libserver` and `libengine2` live /// in different address spaces and comparing them would merge unrelated types. /// /// Returns `(named, propagated, conflicts)`. A conflict is a group whose members Valve names /// differently — which would mean the destructor grouping and the published naming disagree about what /// is the same type, so NOTHING in that group is named. Measured 0 on both games across 44 and 41 /// groups, which is what makes this a standing two-sided oracle rather than a one-way lookup: the names /// come from Valve and the grouping from the binary, and neither reader knows the other exists. fn name_ehandle_classes( bindings: &mut BTreeMap, table: &EhandleClasses, ) -> (u32, u32, Vec) { // (library, token) -> every handle parameter proven to be that type let mut groups: BTreeMap<(String, u64), Vec<(String, bool, usize)>> = BTreeMap::new(); for (name, b) in bindings.iter() { for (is_ret, list) in [(false, &b.params), (true, &b.returns)] { for (i, p) in list.iter().enumerate() { if p.ty == PVAL_EHANDLE && p.type_token != 0 { groups .entry((b.library.clone(), p.type_token)) .or_default() .push((name.clone(), is_ret, i)); } } } } let (mut named, mut propagated) = (0u32, 0u32); let mut conflicts = Vec::new(); for ((lib, token), members) in &groups { let stated: BTreeSet<&str> = members .iter() .filter_map(|(n, is_ret, i)| { let p = &bindings[n]; let pname = &if *is_ret { &p.returns } else { &p.params }[*i].name; table.classes.get(n)?.get(pname).map(String::as_str) }) .collect(); match stated.len() { 0 => {} 1 => { let cls = *stated.iter().next().expect("exactly one"); // `*` is Valve saying "any entity" — a statement, and one that must not be written out // as though it were a class name. if cls == "*" { continue; } for (n, is_ret, i) in members { let b = bindings.get_mut(n).expect("member came from this map"); let p = &mut if *is_ret { &mut b.returns } else { &mut b.params }[*i]; let known = table.classes.get(n).and_then(|m| m.get(&p.name)).is_some(); p.entity_class = Some(cls.to_string()); if known { named += 1; } else { propagated += 1; } } } _ => conflicts.push(format!( "{lib}+{token:#x}: Valve names this one type {:?}", stated.iter().collect::>() )), } } (named, propagated, conflicts) } /// Read Valve's OWN in-binary tables — the highest-ranking naming source the deriver has — folding the /// names they LOCATE and shipping the surface they merely DOCUMENT. /// /// Unlike every other fold input this reads no external file: the tables travel INSIDE each library, so /// the stage re-runs on every build with nothing to maintain and nothing to bootstrap. /// /// Only the entity-IO datadesc yields locators (see [`valvetab::names`] for why the Pulse table does /// not); each of its names gets a fresh `make_sig` at the address Valve paired it with, plus a MEASURED /// argument footprint rather than the placeholder a declared-only source can offer. The Pulse bindings /// become the registry artifact instead. /// /// Folding runs BEFORE the extrapolation fold so ground truth from THIS binary outranks inference — a /// name the binary supplies itself beats the same name guessed by cross-game transfer or the naming pass. fn fold_valve_tables( prof: &GameProfile, build: &Path, source_build: &str, sig_cap: usize, t3: &mut BTreeMap, prov: &mut BTreeMap, ) -> ValveTables { let mut out = ValveTables { abi: BTreeMap::new(), bindings: model::Bindings { meta: model::BindingsMeta { game_key: prof.game_key.to_string(), source_build: source_build.to_string(), pulse: 0, pulse_typed: 0, pulse_callable: 0, entity_inputs: 0, entity_outputs: 0, entity_classes: 0, commands: 0, convars: 0, vscript: 0, vscript_located: 0, vscript_classed: 0, }, pulse: BTreeMap::new(), entity_inputs: Vec::new(), entity_outputs: Vec::new(), entity_classes: BTreeMap::new(), commands: Vec::new(), convars: Vec::new(), vscript: Vec::new(), }, }; let (mut folded, mut ambiguous, mut unmakeable) = (0u32, 0usize, 0u32); let (mut cmd_total, mut cmd_folded, mut cmd_dup, mut cmd_unmakeable) = (0u32, 0u32, 0u32, 0u32); let mut vscript_folded = 0u32; let (mut cmd_measured, mut cmd_agree) = (0u32, 0u32); let (mut io_measured, mut io_agree) = (0u32, 0u32); let (mut out_resolved, mut out_total) = (0u32, 0u32); let (mut pulse_typed, mut pulse_total) = (0u32, 0u32); let mut pulse_stride: Vec<(String, u64, usize, usize)> = Vec::new(); let (mut recv_agree, mut recv_total) = (0u32, 0u32); let (mut dd_qualified, mut dd_total) = (0u32, 0u32); let mut dup_regs = 0u32; let mut dup_conflicts: Vec = Vec::new(); let mut dup_names: BTreeSet = BTreeSet::new(); let mut unreadable: Vec = Vec::new(); let threads = default_threads(None); for f in prof.libs { // A profile's lib list is a superset across builds and games, so an ABSENT library is normal and // silent. A library that is present and does not PARSE is not: it would drop that module's whole // declared surface — its commands, its Pulse bindings, its datadesc — and the artifact would look // exactly like a build where the module registered nothing. Say which one it was. let img = match find_file(build, f, 8) { None => continue, Some(path) => match CodeImage::load(&path) { Ok(img) => img, Err(e) => { unreadable.push(format!("{f} ({e:#})")); continue; } }, }; // Console commands come first because they are the one source here that is independent of the // static tables: a library can register a hundred commands and hold neither a Pulse binding nor // a datadesc array (libsoundsystem does exactly that), so this must not sit behind the // table-emptiness skip below. let commands = concmd::console_commands(&img); let lib = lib_name_from_file(f); // ConVars: the other half of the console surface, read by the same pass. Documentation, not a // locator — a consumer finds a convar by name at runtime; the flags are what it cannot get itself. out.bindings .convars .extend( concmd::convars(&img, &lib) .into_iter() .map(|c| model::ConVar { name: c.name, library: c.library, description: c.description, flags: c.flags, flags_raw: c.flags_raw, addr: c.addr, }), ); { let addrs: Vec = commands.iter().map(|c| c.handler).collect(); let sigs = parallel_map(&addrs, threads, |&a| emit::make_sig(&img, a, sig_cap)); let shapes = parallel_map(&addrs, threads, |&a| abi::abi_shape(&img, a)); for ((c, sig), shape) in commands.iter().zip(sigs).zip(shapes) { cmd_total += 1; // The oracle runs over EVERY command, including the ones that do not fold — it is about // the measurement, not the name. if let Some(s) = shape { cmd_measured += 1; cmd_agree += u32::from( s.int_args <= concmd_int_args(c.form) && s.float_args == 0 && !s.stack_args && s.ret_class != abi::RetClass::ByValue, ); } let key = concmd_key(&c.name); out.bindings.commands.push(model::ConsoleCommand { name: c.name.clone(), library: lib.clone(), description: c.description.clone(), flags: concmd::flag_names(c.flags) .into_iter() .map(str::to_string) .collect(), flags_raw: format!("{:#x}", c.flags), form: c.form.describe().to_string(), abi: shape.map(shipped_abi), addr: format!("{:#x}", c.handler), }); if t3.contains_key(&key) { // A name registered by several libraries keeps the first, in the profile's // resolution order — the same rule the datadesc names follow. cmd_dup += 1; continue; } let Some(sig) = sig else { cmd_unmakeable += 1; continue; }; t3.insert(key.clone(), model::Entry::signature(lib.clone(), sig)); prov.insert( key, model::Provenance { addr: Some(format!("{:#x}", c.handler)), confidence: Some("high".to_string()), // The binary states the name and the address in the same instruction sequence. self_named: Some(true), // The FORM rides in the source id (the `contribution:` shape), because // the prototype manifest needs it: a direct callback takes two arguments and // the two object forms take a receiver as well. See // `prototypes::concommand_contract`. source: Some(format!("{VALVE_CONCOMMAND}:{}", c.form.describe())), rationale: Some(format!( "registered as console command {:?} in {f} ({} callback)", c.name, c.form.describe() )), ..model::Provenance::with_tier(model::Tier::ValveTable) }, ); if let Some(s) = shape { out.abi.insert(concmd_key(&c.name), shipped_abi(s)); } cmd_folded += 1; } } // VScript — the script VM's registry. Folded exactly like the console commands: the binary // states the name and the implementation in one initialiser, so the entry is `valve-table` // provenance and self-named. // // Keyed by the C++ name, not the script-facing one. The script name is what a Lua author types // and belongs in the binding row; the C++ name is what actually lives at the address, and // gamedata names functions. The two never collide with existing entries — measured at ZERO // shared addresses and ZERO shared names against the catalogue, because a // `Script_TakeDamage` is a wrapper and not the `TakeDamage` it wraps. { let vsf = crate::vscript::vscript_functions(&img); // A C++ name registered at more than one address cannot be keyed honestly, so it is dropped // rather than resolved by fiat — the rule the ambiguous datadesc handlers already follow. On // the current builds this is 2 of 1,650 on Dota and 0 of 256 on CS2. let mut addrs_of: HashMap<&str, HashSet> = HashMap::new(); for f in &vsf { if let Some(crate::vscript::Impl::Addr(a)) = f.imp { addrs_of.entry(f.cpp_name.as_str()).or_default().insert(a); } } let foldable: Vec<&crate::vscript::VScriptFunc> = vsf .iter() .filter(|f| { matches!(f.imp, Some(crate::vscript::Impl::Addr(_))) && addrs_of .get(f.cpp_name.as_str()) .is_some_and(|s| s.len() == 1) }) .collect(); let vaddrs: Vec = foldable .iter() .filter_map(|f| match f.imp { Some(crate::vscript::Impl::Addr(a)) => Some(a), _ => None, }) .collect(); let vsigs = parallel_map(&vaddrs, threads, |&a| emit::make_sig(&img, a, sig_cap)); for (f, sig) in foldable.iter().zip(vsigs) { let Some(sig) = sig else { continue }; if t3.contains_key(&f.cpp_name) { continue; } let Some(crate::vscript::Impl::Addr(addr)) = f.imp else { continue; }; t3.insert( f.cpp_name.clone(), model::Entry::signature(lib.clone(), sig), ); prov.insert( f.cpp_name.clone(), model::Provenance { addr: Some(format!("{addr:#x}")), confidence: Some("high".to_string()), self_named: Some(true), source: Some(VALVE_VSCRIPT.to_string()), rationale: Some(format!( "registered with the script VM as {:?} in {f_lib}", f.name, f_lib = lib )), ..model::Provenance::with_tier(model::Tier::ValveTable) }, ); vscript_folded += 1; } for f in &vsf { out.bindings.vscript.push(model::VScriptBinding { name: f.name.clone(), // Filled in by the live oracle; not derivable here (see `VScriptBinding::class`). class: None, cpp: f.cpp_name.clone(), library: lib.clone(), description: f.description.clone().unwrap_or_default(), ret: f.ret.map(str::to_string), ret_raw: f.ret_raw, addr: match f.imp { Some(crate::vscript::Impl::Addr(a)) => Some(format!("{a:#x}")), _ => None, }, vtable_slot: match f.imp { Some(crate::vscript::Impl::Slot(s)) => Some(s), _ => None, }, // Joined at view time, off the function record this row folds onto — see // `VScriptBinding::doc`. doc: None, }); } } let pulse = valvetab::pulse_bindings(&img); // The datadesc by ARRAY, so each handler can be attributed to the class that owns it. The // record carries no owning class; the array's FIELD descriptors do, because a `(member, offset)` // pair is something the SchemaSystem states from an entirely different table. Only an // unambiguous fingerprint counts — an array several classes could satisfy names none of them. let arrays = valvetab::datadesc_arrays(&img); // ONE schema enumeration per library, shared by the array join below and the entity-OUTPUT join // further down. It is a full reflection-table walk over a 133 MB image on Dota, so doing it // twice per library across 21 libraries is not a slow path, it is an out-of-memory kill — which // is exactly what an unguarded second call produced. let sch = schema::enumerate_schema(&img); let mut inputs: Vec = Vec::new(); for arr in arrays { let owners: Vec<&schema::SchemaClass> = if arr.fields.is_empty() { Vec::new() } else { sch.iter() .filter(|c| { arr.fields .iter() .all(|(n, o)| c.fields.iter().any(|f| f.name == *n && f.offset == *o)) }) .collect() }; let class = (owners.len() == 1).then(|| owners[0].name.clone()); dd_total += arr.inputs.len() as u32; dd_qualified += u32::from(class.is_some()) * arr.inputs.len() as u32; for mut i in arr.inputs { i.class = class.clone(); inputs.push(i); } } // An output's locator is a member OFFSET, which is meaningless without the class it sits on — // and the SchemaSystem in this same image already states every member's class, offset and type. // Joining the two is the composition that makes the offset usable, and it doubles as a free // per-build oracle: the datadesc and the schema are independent reflection systems, so their // agreement on a shared fact is evidence both readers are still correct. let outputs = valvetab::datadesc_outputs(&img); if !outputs.is_empty() { let mut by_member: HashMap<(&str, i32), Vec<&schema::SchemaClass>> = HashMap::new(); for c in &sch { for fl in &c.fields { by_member .entry((fl.name.as_str(), fl.offset)) .or_default() .push(c); } } for o in outputs { // Only an UNAMBIGUOUS join is used: several classes declaring the same member at the same // offset would put us back to guessing, which is the thing this exists to stop. let hit = by_member .get(&(o.member.as_str(), o.offset as i32)) .filter(|v| v.len() == 1) .map(|v| v[0]); out_resolved += u32::from(hit.is_some()); out_total += 1; out.bindings.entity_outputs.push(model::EntityOutput { output: o.output, member: o.member.clone(), library: lib_name_from_file(f), offset: o.offset, class: hit.map(|c| c.name.clone()), }); } } for e in valvetab::entity_classes(&img) { out.bindings .entity_classes .entry(e.classname) .or_insert(e.class); } if pulse.is_empty() && inputs.is_empty() { continue; } let (names, dropped) = valvetab::names(&inputs); ambiguous += dropped; // One measurement per distinct handler: an address recurs across records (the same handler // serves several inputs). Sorted so the parallel pass is deterministic. let mut addrs: Vec = inputs .iter() .map(|i| i.func) .collect::>() .into_iter() .collect(); addrs.sort_unstable(); let shape_at: HashMap = parallel_map(&addrs, threads, |&a| (a, abi::abi_shape(&img, a))) .into_iter() .filter_map(|(a, s)| s.map(|s| (a, s))) .collect(); // make_sig dominates the cost and is a pure read on the image, so it runs across threads too; // the result order follows the input, so the fold stays deterministic. let name_addrs: Vec = names.iter().map(|n| n.addr).collect(); let sigs = parallel_map(&name_addrs, threads, |&a| emit::make_sig(&img, a, sig_cap)); // The oracle runs over EVERY handler, not just the ones that fold: an ambiguous name is still a // function with a known prototype, and the check is about the measurement, not the name. for i in &inputs { if let Some(s) = shape_at.get(&i.func) { io_measured += 1; io_agree += u32::from(within_io_prototype(s)); } } for (n, sig) in names.iter().zip(sigs) { let shape = shape_at.get(&n.addr).copied(); if t3.contains_key(&n.name) { continue; // an earlier library's table already vouched for this name } let Some(sig) = sig else { unmakeable += 1; continue; }; t3.insert(n.name.clone(), model::Entry::signature(lib.clone(), sig)); prov.insert( n.name.clone(), model::Provenance { addr: Some(format!("{:#x}", n.addr)), confidence: Some("high".to_string()), // The binary names the function itself — a stronger claim than the naming pass's // "the name appears in the function's own bytes", not a weaker one. self_named: Some(true), // DECLARED, not measured. The engine invokes every entity-IO handler through // `void (CBaseEntity::*)(inputdata_t&)`, so the return value is never read — the // prototype settles it. That is worth preferring here because the measured class is // specifically weak on exactly this question: a `void` function using RAX as scratch // reads back as `ret=int`, and across these same handlers only ~12% classify as void. by_value: Some(false), ret_class: Some(abi::RetClass::Void.describe().to_string()), source: Some(VALVE_DATADESC.to_string()), rationale: Some(format!("Valve entity-IO datadesc in {f}")), ..model::Provenance::with_tier(model::Tier::ValveTable) }, ); if let Some(s) = shape { out.abi.insert(n.name.clone(), shipped_abi(s)); } folded += 1; } // The registry describes the callable SURFACE, so it ships every record — including the Pulse // bindings that have no locator and the handlers whose bare name was too ambiguous to fold. An // `InputEnable` on 48 classes is 48 real handlers; only naming them individually is impossible. // // Each binding's two accessors are read for its TYPED SIGNATURE, which is the one thing the // record itself does not carry. Read together rather than one at a time, because the element // stride is derived by consensus across the image — see `pulse::read_all`. let pairs: Vec<(u64, u64)> = pulse .iter() .map(|b| (b.descriptor, b.arg_descriptor)) .collect(); let (sig_of, stride, stride_votes, stride_rivals) = pulse::read_all(&img, &pairs, threads); if stride_votes > 0 { pulse_stride.push((lib.clone(), stride, stride_votes, stride_rivals)); } for (b, sig) in pulse.into_iter().zip(sig_of) { pulse_typed += u32::from(sig.is_some()); pulse_total += 1; let kind = binding_kind(b.flags); // A FREE ORACLE, and a two-sided one. The receiver flag was decoded last session from naming // conventions alone; the parameter list is read from a completely different place in the // binary. If the flag means what it was inferred to mean, an `instance` binding takes a // `_Target` receiver and nothing else does — so agreement is independent evidence for the // flag decoding, and disagreement would mean the signature reader had shifted a parameter // list. Measured 999/999 and 859/859 registrations, no exceptions in either game. if let Some(s) = sig.as_ref() { let has_target = s.args.first().is_some_and(|p| p.name == "_Target"); recv_total += 1; recv_agree += u32::from(has_target == (kind == model::BindingKind::Instance)); } let entry = model::Binding { library: lib.clone(), display: b.display, description: b.description, policy: model::CallPolicy { kind, mutates: b.flags.mutating, blocking: b.flags.blocking, raw: [b.flags.raw.0, b.flags.raw.1], }, params: sig.as_ref().map(|s| s.args.clone()).unwrap_or_default(), returns: sig.as_ref().map(|s| s.returns.clone()).unwrap_or_default(), typed: sig.is_some(), descriptor: format!("{:#x}", b.descriptor), shim: (b.shim != 0).then(|| format!("{:#x}", b.shim)), // Measured per shim rather than assumed from the tier: the calling contract is fixed, but // WHICH slots a given shim reads is the whole difference between host-callable and not. call: (b.shim != 0) .then(|| pulse::shim_reads(&img, b.shim)) .flatten() .map(|r| model::ShimCall { needs: r.needs().to_string(), reads: r.reads.iter().map(|s| s.to_string()).collect(), }), }; // The registry is keyed by qualified name across libraries, so a binding registered by // more than one module keeps ONE row. That is the documented lossiness — of the LIBRARY, // not of the signature — and it is checked rather than assumed: every duplicate is compared // against the row already present and a disagreement is reported. // // IT IS REPORTING, on every derive of both games — CS2 flags 331 of 419 repeat // registrations and Dota 271 of 359. So the artifact IS choosing between two accounts of // one binding for those names, which is exactly the outcome this check exists to make // impossible to miss. An earlier "0 disagreements" reading here predates the typed-signature // recovery and was never revisited. Until it is, a consumer reading `params`/`returns` for a // multiply-registered binding is reading ONE module's account of it, not a merged one. // The FIRST library wins, which is the rule every sibling table in this loop already // follows (commands, VScript, datadesc, entity classes, the live schema) and the one // `GameProfile::libs` documents — "an earlier lib wins". Pulse was the lone `insert`, so the // LAST registration overwrote everything, and 224 of 580 CS2 rows shipped another module's // account under a name libserver also registers. It also broke the oracle pairing: the live // shim and descriptor checks run against the SERVER image, so for exactly the duplicated // names the rows that got verified were not the rows that got shipped. // // One exception, because precedence must not cost information: a typed row is strictly more // than an untyped one, so a later library that DID recover a signature replaces an earlier // one that did not. Never the reverse. match out.bindings.pulse.get(&b.name) { Some(prev) => { dup_regs += 1; dup_names.insert(b.name.clone()); if prev.params != entry.params || prev.returns != entry.returns || prev.typed != entry.typed { dup_conflicts.push(b.name.clone()); } if entry.typed && !prev.typed { out.bindings.pulse.insert(b.name, entry); } } None => { out.bindings.pulse.insert(b.name, entry); } } } for i in inputs { out.bindings.entity_inputs.push(model::EntityInput { input: i.io_name, class: i.class, handler: i.handler, library: lib.clone(), abi: shape_at.get(&i.func).copied().map(shipped_abi), addr: format!("{:#x}", i.func), }); } } out.bindings .entity_inputs .sort_by(|a, b| (&a.input, &a.handler, &a.addr).cmp(&(&b.input, &b.handler, &b.addr))); out.bindings.meta.pulse = out.bindings.pulse.len(); out.bindings.meta.pulse_typed = out.bindings.pulse.values().filter(|b| b.typed).count(); // Counted from the emitted rows rather than tallied during the fold, so the number in `meta` cannot // drift from the number of rows a consumer can actually act on. out.bindings.meta.pulse_callable = out .bindings .pulse .values() .filter(|b| b.call.as_ref().is_some_and(|c| c.needs == "args-only")) .count(); out.bindings.meta.entity_inputs = out.bindings.entity_inputs.len(); out.bindings .entity_outputs .sort_by(|a, b| (&a.output, &a.member).cmp(&(&b.output, &b.member))); out.bindings.meta.entity_outputs = out.bindings.entity_outputs.len(); out.bindings.meta.entity_classes = out.bindings.entity_classes.len(); out.bindings .commands .sort_by(|a, b| (&a.name, &a.addr).cmp(&(&b.name, &b.addr))); out.bindings.meta.commands = out.bindings.commands.len(); out.bindings.meta.convars = out.bindings.convars.len(); out.bindings.meta.vscript = out.bindings.vscript.len(); out.bindings.meta.vscript_located = vscript_folded as usize; if cmd_total > 0 { let libs: BTreeSet<&str> = out .bindings .commands .iter() .map(|c| c.library.as_str()) .collect(); eprintln!( " +{cmd_folded} names from console-command registration ({cmd_total} commands across \ {} libraries)", libs.len() ); if cmd_dup > 0 || cmd_unmakeable > 0 { eprintln!( " skipped: {cmd_dup} already named by an earlier library, {cmd_unmakeable} \ unmakeable-sig" ); } eprintln!( " +{vscript_folded} names from the script-VM registry ({} bindings, {} with a \ script-facing name Valve documents)", out.bindings.vscript.len(), out.bindings .vscript .iter() .filter(|v| !v.description.is_empty()) .count() ); // A STANDING ORACLE, and an intrinsic one — it needs no external list. Every command has its // own handler, so commands and distinct handler addresses should track each other. The failure // this catches is specific: the member-callback form finds its handler by scanning the accessor // object, and a layout change there would start returning the shared dispatch thunk instead of // the per-command function. That would still resolve, still validate live, and still be wrong — // and it would show up here as a collapse in the second number while the first held steady. let distinct: BTreeSet<&str> = out .bindings .commands .iter() .map(|c| c.addr.as_str()) .collect(); let by_form = |w: &str| out.bindings.commands.iter().filter(|c| c.form == w).count(); eprintln!( " console-command handler distinctness: {} commands -> {} distinct handlers ({} \ direct, {} interface, {} member)", cmd_total, distinct.len(), by_form("direct"), by_form("interface"), by_form("member"), ); if cmd_measured > 0 { eprintln!( " console-command ABI oracle: {cmd_agree}/{cmd_measured} handlers measure within \ the callback prototype their registration declares ({:.1}%)", 100.0 * f64::from(cmd_agree) / f64::from(cmd_measured) ); } } eprintln!(" +{folded} names from Valve's entity-IO datadesc"); if ambiguous > 0 || unmakeable > 0 { eprintln!( " skipped: {ambiguous} ambiguous (one name, several addresses — no owning class to \ disambiguate), {unmakeable} unmakeable-sig" ); } if io_measured > 0 { eprintln!( " entity-IO ABI oracle: {io_agree}/{io_measured} handlers measure within the declared \ void(2 int) footprint ({:.1}%)", 100.0 * f64::from(io_agree) / f64::from(io_measured) ); } if out_total > 0 { eprintln!( " entity-output schema join: {out_resolved}/{out_total} outputs resolved to exactly one \ class ({:.1}%) — two independent reflection systems agreeing on the same member", 100.0 * f64::from(out_resolved) / f64::from(out_total) ); } // OUTSIDE the Pulse block below, and that is exactly the point: a library that failed to parse // contributes no Pulse registrations, so `pulse_total > 0` is the condition LEAST likely to hold when // this warning is most needed. Nested there, it silenced itself in its own trigger case. if !unreadable.is_empty() { eprintln!( " WARNING {} declared-surface librar{} PRESENT but unreadable, so their commands, Pulse \ bindings and datadesc are missing from this artifact rather than absent from the build: {}", unreadable.len(), if unreadable.len() == 1 { "y is" } else { "ies are" }, unreadable.join(", ") ); } if pulse_total > 0 { // The stride is DERIVED per image, so report it: it is the one layout fact the reader takes from // consensus rather than from each record, and a build that changed the record size would show up // here as a new number before it showed up as lost signatures. let strides: BTreeSet = pulse_stride.iter().map(|(_, s, _, _)| *s).collect(); let votes: usize = pulse_stride.iter().map(|(_, _, v, _)| v).sum(); let rivals: usize = pulse_stride.iter().map(|(_, _, _, r)| r).sum(); // REGISTRATIONS, not distinct bindings: a binding registered by several libraries is read once // per library, so this population is larger than `meta.pulse`. Saying "bindings" here would put // two different numbers under one word. eprintln!( " Pulse typed signatures: {pulse_typed}/{pulse_total} registrations recovered from their \ own descriptor initializers ({:.1}%); element stride {} agreed by {votes} records across \ {} libs, {rivals} dissenting", 100.0 * f64::from(pulse_typed) / f64::from(pulse_total), strides .iter() .map(u64::to_string) .collect::>() .join("/"), pulse_stride.len() ); eprintln!( " Pulse receiver cross-check: {recv_agree}/{recv_total} registrations agree on whether \ they take a receiver ({:.1}%) — the `+56` flag byte and the recovered parameter list are \ read from different places and neither knows about the other", 100.0 * f64::from(recv_agree) / f64::from(recv_total.max(1)) ); if dup_conflicts.is_empty() { eprintln!( " Pulse multi-library duplicates: {dup_regs} repeat registrations across {} \ bindings, all agreeing with the row already read", dup_names.len() ); } else { eprintln!( " FLAG {} of {dup_regs} repeat registrations DISAGREE on their signature — the \ registry keeps one row per name, so one account is being discarded: {}", dup_conflicts.len(), dup_conflicts.join(", ") ); } } eprintln!( " entity-IO handlers class-qualified by their array's own field descriptors: {dd_qualified}/{dd_total} \ (the schema and the datadesc are independent reflection systems; an array whose fingerprint fits \ several classes or none names neither)" ); println!( " binding registry: {} Pulse bindings, {} entity-IO inputs, {} outputs, {} entity classnames", out.bindings.meta.pulse, out.bindings.meta.entity_inputs, out.bindings.meta.entity_outputs, out.bindings.meta.entity_classes ); out } /// The Valve-table stage's products beyond the locators it folds into `t3`. pub(crate) struct ValveTables { /// Argument footprints measured for the names it folded, for the monolith's `abi` field. abi: BTreeMap, /// The declared callable surface, shipped as its own artifact. bindings: model::Bindings, } /// The receiver shape a Pulse binding's flags describe. The two bytes are mutually exclusive in the /// data; `Cell` is the neither case — a cell entry point the VM drives, not a graph-author binding. fn binding_kind(f: valvetab::PulseFlags) -> model::BindingKind { match (f.library, f.instance) { (true, false) => model::BindingKind::Library, (false, true) => model::BindingKind::Instance, _ => model::BindingKind::Cell, } } /// Fold string anchors onto every monolith entry whose name carries one, in every tier. /// /// Returns how many landed. That number is REPORTED rather than assumed because the two populations are /// independent: the catalogue says which names have anchors, the derive says which names got a locator, and /// an anchor for a name that never resolved has nowhere to go. A large gap is a fact about the build, not a /// bug — but it should be visible rather than inferred from an artifact diff. fn attach_anchors(mono: &mut model::Monolith, anchors: &BTreeMap>) -> usize { let mut n = 0; for tier in [ &mut mono.core, &mut mono.high_confidence, &mut mono.experimental, ] { for (name, e) in tier.iter_mut() { if let Some(a) = anchors.get(name) { // Deduplicated on the way in: the same anchor can appear on several catalogue variants, // and this list ships in a byte-reproducible artifact. for s in a { if !e.locator.anchors.contains(s) { e.locator.anchors.push(s.clone()); } } n += 1; } } } n } /// A string worth anchoring on: long enough to be distinctive, printable, and not a lone format specifier. /// /// The thresholds are the knob this whole feature turns on. Loosening them raises coverage and lowers /// distinctiveness; they were measured, not guessed — at these values 4,322 of libserver's 70,288 functions /// have a unique anchor, and 27% of the shipped set does. fn usable_anchor(s: &str) -> bool { s.len() >= 8 && s.len() <= 200 && s.is_ascii() && s.chars().filter(|c| c.is_ascii_alphanumeric()).count() >= 5 } /// Where every anchorable string in `img` is referenced FROM: string address -> the instruction addresses /// that load it, plus the string itself. /// /// Deliberately instruction-level and function-agnostic. Attributing a string to a function needs function /// BOUNDARIES, and this binary does not reliably supply them — `.eh_frame_hdr` describes 8,327 of libserver's /// ~70,000 functions, so a `[entry, next_entry)` range routinely spans a real function plus one or more /// unindexed neighbours, and every neighbour's strings then look like the first function's. An instruction /// address, by contrast, is exactly what it is. The caller decides membership against an extent it walked /// itself, which is the only claim available that does not depend on the entry list being complete. fn string_refs(img: &CodeImage) -> HashMap)> { let entries = crate::locate::function_entries(img); let mut out: HashMap)> = HashMap::new(); for (i, &start) in entries.iter().enumerate() { let end = entries.get(i + 1).copied().unwrap_or(u64::MAX); let Some(code) = img.code_range(start, end) else { continue; }; let mut insn = iced_x86::Instruction::default(); let mut dec = iced_x86::Decoder::with_ip(64, code, start, iced_x86::DecoderOptions::NONE); while dec.can_decode() { dec.decode_out(&mut insn); if insn.is_invalid() || !insn.is_ip_rel_memory_operand() { continue; } let va = insn.ip_rel_memory_address(); if let Some(e) = out.get_mut(&va) { e.1.push(insn.ip()); } else if let Some(s) = img.read_c_string(va).filter(|s| usable_anchor(s)) { out.insert(va, (s, vec![insn.ip()])); } } } out } /// Instruction addresses reachable from `entry` by following control flow, and the string addresses it loads. /// /// The function's OWN extent, determined by where its branches go and where it returns, rather than by the /// next symbol. That is what makes the anchor check sound without a complete function list. fn reachable_strings(img: &CodeImage, entry: u64) -> Option<(HashSet, Vec)> { const CAP: u64 = 0x4000; let all = img.code_at(entry)?; let extent = (all.len() as u64).min(CAP); let code = &all[..extent as usize]; let mut seen: HashSet = HashSet::new(); let mut loads: Vec = Vec::new(); // Saturating for the same reason `pulse::shim_reads` is: a file-controlled extent must not wrap the // range inside out under the overflow-checked build the fuzzers use. let end = entry.saturating_add(extent); let mut work = vec![entry]; let mut insn = iced_x86::Instruction::default(); while let Some(at) = work.pop() { if at < entry || at >= end || !seen.insert(at) || seen.len() > 40000 { continue; } let mut dec = iced_x86::Decoder::with_ip( 64, &code[(at - entry) as usize..], at, iced_x86::DecoderOptions::NONE, ); if !dec.can_decode() { continue; } dec.decode_out(&mut insn); if insn.is_invalid() || insn.len() == 0 { continue; } if insn.is_ip_rel_memory_operand() { loads.push(insn.ip_rel_memory_address()); } match insn.flow_control() { iced_x86::FlowControl::Return | iced_x86::FlowControl::IndirectBranch | iced_x86::FlowControl::Exception | iced_x86::FlowControl::Interrupt => {} iced_x86::FlowControl::UnconditionalBranch => work.push(insn.near_branch_target()), iced_x86::FlowControl::ConditionalBranch => { work.push(at + insn.len() as u64); work.push(insn.near_branch_target()); } _ => work.push(at + insn.len() as u64), } } Some((seen, loads)) } /// Derive an anchor for every entry that has none, from the address its SHIPPED signature resolves to. /// /// Three conditions, each closing a way this can name the wrong function: /// /// 1. **The resolved address must be a function ENTRY POINT.** ModSharp's `refs.strings` locates a /// *function*; a great many shipped locators deliberately point MID-function (`CBaseButton::InputPress` /// resolves to a `mov`, `BotNavIgnore` to a `je` — patterns anchored at a hook site, not a prologue). An /// anchor cannot denote the same thing as one of those, so those entries get none rather than a locator /// that resolves somewhere else. /// 2. **The string must be referenced from inside the function's OWN flow-reachable code**, walked from the /// entry, not from a `[entry, next_entry)` range. `.eh_frame_hdr` covers a small fraction of these /// binaries' functions, so such a range routinely swallows unindexed neighbours and inherits their /// strings — which is exactly how a first cut of this produced "`CBaseButton::InputPress` references /// *Traced intervals in %.3fus*". /// 3. **Every instruction that loads the string must be inside that same reachable set.** This is the /// uniqueness test, done at instruction level so it never consults a function boundary. A string also /// loaded from elsewhere locates nothing and is dropped. /// /// Server-library only, the restriction [`recover_by_string_anchor`] already carries: the fold holds that one /// image, and loading a second full set of 22 keyed images to reach the rest would add several hundred MB to a /// pipeline that has already been OOM-killed on Dota. /// /// Returns `(attached, considered)`. Degrades quietly toward FEWER anchors and never toward a wrong one. fn attach_derived_anchors( mono: &mut model::Monolith, img: &CodeImage, server_lib: &str, ) -> (usize, usize) { let wants = |e: &model::MonoEntry| { e.locator.anchors.is_empty() && e.locator .signature .as_ref() .is_some_and(|s| s.library == server_lib) }; let any = [&mono.core, &mono.high_confidence, &mono.experimental] .iter() .any(|t| t.values().any(&wants)); if !any { return (0, 0); } let refs = string_refs(img); // Condition 1's test set: the addresses this image treats as function starts. let starts = crate::locate::function_entries(img); // Why each candidate was rejected, so a low yield is a FACT rather than a mystery. The three // conditions fail for very different reasons and the mix differs sharply between games (CS2 derives // ~5% of candidates, Dota ~0.5%); without this the difference is unattributable. let (mut attached, mut considered) = (0usize, 0usize); let (mut no_resolve, mut mid_fn, mut no_unique) = (0usize, 0usize, 0usize); for tier in [ &mut mono.core, &mut mono.high_confidence, &mut mono.experimental, ] { for e in tier.values_mut() { if !wants(e) { continue; } considered += 1; let Some(sig) = e.locator.signature.as_ref() else { continue; }; let Ok(pat) = crate::sig::Pattern::parse(&sig.linux) else { continue; }; let hits = img.find(&pat); let [addr] = hits.as_slice() else { no_resolve += 1; continue; }; let addr = *addr; if starts.binary_search(&addr).is_err() { mid_fn += 1; // condition 1: mid-function locator, not a function an anchor can name continue; } let Some((reach, loads)) = reachable_strings(img, addr) else { continue; }; // Candidates this function actually loads, longest first then lexicographic — deterministic, // because this lands in a byte-reproducible artifact. let mut cands: Vec<&(String, Vec)> = loads .iter() .filter_map(|va| refs.get(va)) .filter(|(_, from)| from.iter().all(|ip| reach.contains(ip))) .collect(); cands.sort_unstable_by(|a, b| b.0.len().cmp(&a.0.len()).then_with(|| a.0.cmp(&b.0))); match cands.first() { Some((s, _)) => { e.locator.anchors.push(s.clone()); attached += 1; } None => no_unique += 1, } } } eprintln!( " anchor derivation: {attached} attached; rejected {no_resolve} (pattern did not resolve \ uniquely), {mid_fn} (locator is mid-function, which an anchor cannot name), {no_unique} (no string \ unique to the function)" ); (attached, considered) } /// Assemble the monolith in memory and render its CS# gamedata (the string the live validate stage checks). /// Writes NOTHING — `produce` holds the `Monolith` (to annotate it live) plus this render, and writes the /// monolith exactly once at the end (after live validation, if a game is present). fn build_monolith( prof: &GameProfile, source_build: &str, version: &str, core_json: &str, flagged: &[model::Flagged], experimental: Option<&[ExpGuess]>, unverified: &BTreeSet, t3: &BTreeMap, prov: &BTreeMap, abi: &BTreeMap, anchors: &BTreeMap>, server_img: Option<(&CodeImage, &str)>, ) -> Result<(model::Monolith, String)> { let mut mono = assemble_monolith( prof, source_build, version, core_json, flagged, experimental, unverified, t3, prov, abi, )?; // Attached AFTER assembly, across every tier at once, rather than at the three MonoEntry construction // sites: an anchor belongs to a NAME, not to a tier, and one pass cannot leave a tier out by omission. let attached = attach_anchors(&mut mono, anchors); // Then DERIVE one for everything the catalogue does not cover. Reported separately from the catalogued // count: they are different claims — one is a curated string somebody chose, the other is this build's // own machine code answering the same question — and collapsing them would hide either going to zero. let (derived, considered) = match server_img { Some((img, lib)) => attach_derived_anchors(&mut mono, img, lib), None => (0, 0), }; eprintln!( " string anchors: {attached} of {} catalogued reached the monolith; {derived} DERIVED for {considered} \ server entries that had none ({} total anchored)", anchors.len(), attached + derived ); let cssharp = render::render_monolith_cssharp(&mono, model::TierSelect::HighConfidence); eprintln!( " monolith: {} core + {} high-conf + {} experimental + {} unresolved", mono.meta.counts.core, mono.meta.counts.high_confidence, mono.meta.counts.experimental, mono.meta.counts.unresolved ); eprintln!( " aliases: {} names over {} functions carry a second shipped name ({:.1}% of the resolved \ surface is one function under several names)", mono.meta.aliased_names, mono.meta.alias_groups, 100.0 * mono.meta.aliased_names as f64 / (mono.meta.counts.core + mono.meta.counts.high_confidence).max(1) as f64 ); Ok((mono, cssharp)) } /// The fold's inputs, grouped so a new fold input is added in ONE place — not threaded through a long /// positional call where two same-typed `Option<&Path>` are a silent transposition hazard. Built by the /// `produce` orchestration; the `--extra-*`/`--full-names` members are the multilib/experimental opt-ins. pub(crate) struct FoldArgs<'a> { pub build: &'a Path, pub lib: &'a str, /// Optional bring-up refinements (which extrapolated names may promote, and their prefiltered /// context). `None` = empty; a game derives its full catalogue without either. pub promotable: Option<&'a Path>, pub candidates: Option<&'a Path>, /// The derive's in-memory rendered core gamedata + its flag list (`None` when empty) — handed straight /// from `gamedata()`, no disk round-trip. pub core: &'a str, pub flagged: &'a [model::Flagged], /// Core names whose fingerprint check rejected every candidate (see [`Derived::unverified`]) — marked /// `catalogue-unverified` in the monolith rather than passed off as fingerprint-verified. pub unverified: &'a BTreeSet, /// The derive's measured argument footprints, folded onto the monolith entries. pub abi: &'a BTreeMap, /// The derive's catalogue string anchors, folded onto the monolith entries the same way. pub anchors: &'a BTreeMap>, pub sig_cap: usize, pub version: &'a str, pub full_names: Option<&'a Path>, pub extra_offsets: Option<&'a Path>, pub extra_sigs: Option<&'a Path>, /// Valve's naming for the entity class behind each `PVAL_EHANDLE` parameter /// (`mappings/ehandle-classes.json`). Static repo input, optional: without it the bindings artifact /// states no class, which is what it stated before this existed. pub ehandle_classes: Option<&'a Path>, pub source_build: &'a str, } /// Measure the argument footprint of OFFSET-located entries. A vtable slot IS the function address once the /// class's vtable is located, so the same `abi_shape` that covers signatures covers virtuals — and virtuals /// are the bulk of the shipped surface. Signature entries already carry a shape from the derive; this fills /// the other half. /// /// Searches EVERY server-mapped library, not just the default one: an extrapolated name routinely sits on a /// class owned by `engine2`/`tier0`/etc., and looking only in the game lib silently skips those. Vtables are /// resolved once per class, since one class backs many entries. fn offset_abi_shapes<'a>( prof: &GameProfile, imgs: &[CodeImage], entries: impl Iterator, ) -> BTreeMap { let mut out = BTreeMap::new(); let mut vts: HashMap> = HashMap::new(); for (name, class, slot) in entries { if class.is_empty() || class == name { continue; // a free function has no vtable to index } let hit = vts.entry(class.to_string()).or_insert_with(|| { imgs.iter().enumerate().find_map(|(i, im)| { rtti::find_vtable(im, class, prof.max_vtable_slots).map(|vt| (i, vt)) }) }); let (Some((li, vt)), Ok(idx)) = (hit.as_ref(), usize::try_from(slot)) else { continue; }; if let Some(&addr) = vt.slots.get(idx) && let Some(sh) = abi::abi_shape(&imgs[*li], addr) { out.insert(name.to_string(), shipped_abi(sh)); } } out } /// Fold the verified name-extrapolation harvest + curated entries under the guaranteed core into ONE /// shipped gamedata — the whole deterministic promote stage in a single per-game command. Each promotable /// gets the correct-by-construction locator (a vtable OFFSET when its RTTI class is clean and the AI's /// class matches ground-truth, else a fresh `make_sig`); the prefiltered `--candidates` membership drops /// dead weight; the guaranteed `--core` wins name collisions. `--extra-offsets` folds multilib ground-truth /// vtable methods (name + slot) from the other server libs directly as high_confidence offsets. Emits the /// combined file + a provenance sidecar (tier / confidence / return-class / by-value flag). pub(crate) fn build_gamedata_cmd(prof: &GameProfile, a: FoldArgs) -> Result { let FoldArgs { build, lib, promotable, candidates, core, flagged, unverified, abi, anchors, sig_cap, version, full_names, extra_offsets, extra_sigs, ehandle_classes, source_build, } = a; let img = load_lib(build, lib)?; let default_lib = lib_name_from_file(lib); // Both are OPTIONAL bring-up inputs: they refine which extrapolated names get promoted, and a game // with neither still derives its whole catalogue. Absent = empty, so standing up a new game needs only // a catalogue rather than two files with no generator in the repo. let promo: Vec = match promotable { Some(p) => { serde_json::from_str(&std::fs::read_to_string(p)?).context("parse promotable json")? } None => Vec::new(), }; let cdoc: CandDoc = match candidates { Some(p) => { serde_json::from_str(&std::fs::read_to_string(p)?).context("parse candidates json")? } None => CandDoc::default(), }; let cand: HashMap = cdoc .candidates .into_iter() .map(|c| (c.addr.clone(), c)) .collect(); // Protobuf message types are dead weight even when the class name (e.g. `AccountActivity`, // `CGCToGCMsgMasterAck`) matches no foreign/CMsg prefix — only the serializer METHOD names reveal them. // Cluster over the promoted set (the names this fold would ship). let protobuf_classes = protobuf_message_classes( prof, promo.iter().filter(|p| p.promote).map(|p| p.name.as_str()), ); let mut t3: BTreeMap = BTreeMap::new(); let mut prov: BTreeMap = BTreeMap::new(); // Valve's own in-binary name tables, folded FIRST: the extrapolation loop below skips any name `t3` // already holds, so folding here is what ranks ground truth from this binary above every inferred // name. (`core` still wins everything — its overlap removal runs after both.) let mut valve = fold_valve_tables(prof, build, source_build, sig_cap, &mut t3, &mut prov); // Valve's own naming for the entity class behind each handle, propagated across the parameters this // build proves are the same type. A static repo input like `mappings/prototypes.json` — INPUT, never // folded forward — and optional: without it the artifact simply states no class, which is what it // stated before this existed. if let Some(path) = ehandle_classes { let table: EhandleClasses = serde_json::from_str( &std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?, ) .context("parse ehandle-classes json")?; let (named, propagated, conflicts) = name_ehandle_classes(&mut valve.bindings.pulse, &table); println!( " PVAL_EHANDLE entity classes: {named} named from Valve's metadata, {propagated} more \ propagated across parameters the destructor address proves are the same type" ); // Two-sided: the names come from Valve's dump, the grouping from the binary's destructor // addresses, and neither reader knows the other exists. Agreement is evidence for both. if conflicts.is_empty() { println!(" PVAL_EHANDLE grouping vs Valve's naming: no group carries two classes"); } else { for c in &conflicts { println!(" EHANDLE CONFLICT {c}"); } } } let (mut n_off, mut n_sig, mut n_byval) = (0u32, 0u32, 0u32); let (mut dead, mut unmakeable) = (0u32, 0u32); let mut unmakeable_list = Vec::new(); // The fold's make_sig dominates the cost and is a pure read on `img`, so do it ONCE in // parallel for every promoted addr that routes to a SIGNATURE (not a vtable offset); the loop // below just looks the result up. make_sig is a pure fn of (img, addr, cap), so the fold's // output is byte-identical to the serial version — this changes WHERE the work runs, not what // it produces. (Inherited methods, whose declaring class != the candidate's RTTI class, fall to // make_sig rather than an offset, so this path carries real load.) let sig_addrs: Vec = promo .iter() .filter(|pr| { pr.promote && !is_dead_weight_name(prof, &pr.name) && !is_serializer_plumbing(prof, &pr.name, &protobuf_classes) }) .filter_map(|pr| { let c = cand.get(&pr.addr)?; if is_offset_locator(prof, pr, c) { return None; } parse_hex_addr(&pr.addr) }) .collect::>() .into_iter() .collect(); let sig_by_addr: HashMap = parallel_map(&sig_addrs, default_threads(None), |&a| { (a, emit::make_sig(&img, a, sig_cap)) }) .into_iter() .filter_map(|(a, s)| s.map(|s| (a, s))) .collect(); for pr in &promo { if !pr.promote || t3.contains_key(&pr.name) { continue; } // Dead-weight gates: (1) the resolved NAME (foreign/protobuf functions the class-based dump // prefilter can't see); (2) a lone HARD serializer method or a protobuf message class (>=3 // serializer methods); (3) membership in the prefiltered candidate dump. if is_dead_weight_name(prof, &pr.name) || is_serializer_plumbing(prof, &pr.name, &protobuf_classes) { dead += 1; continue; } let Some(c) = cand.get(&pr.addr) else { dead += 1; continue; }; let ret = ByValClass::from_abi(c.abi.as_deref()); if is_offset_locator(prof, pr, c) { t3.insert(pr.name.clone(), model::Entry::offset(c.slot.unwrap())); n_off += 1; } else { let Some(addr) = parse_hex_addr(&pr.addr) else { unmakeable += 1; unmakeable_list.push(format!("{} (bad addr {:?})", pr.name, pr.addr)); continue; }; match sig_by_addr.get(&addr).cloned() { Some(sig) => { t3.insert( pr.name.clone(), model::Entry::signature(default_lib.to_string(), sig), ); n_sig += 1; } None => { unmakeable += 1; unmakeable_list.push(format!("{} @ {addr:#x}", pr.name)); continue; } } } let tier = if pr.self_named { model::Tier::SelfNamed } else if pr.corroboration == "exact" { model::Tier::DictExact } else { model::Tier::Contextual }; if ret.is_by_value() { n_byval += 1; } // Build the provenance TYPED so the compiler enforces the schema (model::Provenance serializes its // fields in declaration order). prov.insert( pr.name.clone(), model::Provenance { addr: Some(pr.addr.clone()), confidence: Some(pr.confidence.clone()), self_named: Some(pr.self_named), rationale: Some(pr.rationale.clone()), ret_class: Some(format!("ret={}", ret.as_str())), by_value: Some(ret.is_by_value()), source: Some("source2rosetta-nameext".to_string()), rtti_class: c.class.clone(), ..model::Provenance::with_tier(tier) }, ); } // fold under the guaranteed core (core wins any name collision) — parsed from the derive's in-memory // rendered core, no disk round-trip. let core_gd: GdMap = read_gamedata_str(core)?; let overlaps: Vec = t3 .keys() .filter(|k| core_gd.contains_key(*k)) .cloned() .collect(); for k in &overlaps { t3.remove(k); prov.remove(k); } // Multilib ground-truth folds for the non-primary libs (macOS symbol-transfer offsets + AI-name sigs), // each under the same core-wins precedence — see fold_extra_offsets / fold_extra_sigs. if let Some(ep) = extra_offsets { fold_extra_offsets(ep, &core_gd, &mut t3, &mut prov)?; } if let Some(sp) = extra_sigs { fold_extra_sigs(build, sp, sig_cap, &core_gd, &mut t3, &mut prov)?; } let tier_count = |t: model::Tier| prov.values().filter(|p| p.tier == t).count(); let (n_self, n_dict) = ( tier_count(model::Tier::SelfNamed), tier_count(model::Tier::DictExact), ); eprintln!( "produce (fold) {version}: {} core + {} extrapolated = {} functions ({n_off} offsets, {n_sig} sigs; \ {n_self} self-named, {n_dict} dict-exact, {n_byval} by-value-flagged)", core_gd.len(), t3.len(), core_gd.len() + t3.len() ); eprintln!( " {dead} promotable dropped (dead-weight, absent from prefiltered candidates); {unmakeable} unmakeable-sig; {} core-name overlaps dropped", overlaps.len() ); for u in unmakeable_list.iter().take(20) { eprintln!(" UNMAKEABLE {u}"); } // Least-filtered inclusion tier: the experimental band, in memory, emitted iff `--full-names` is given. // Emitted here (the fold step) because this is where names + locators become gamedata entries — the SAME // derivation, just with no promote/dead-weight gate. let experimental = full_names .map(|fnames| emit_experimental_band(prof, &img, build, fnames, &default_lib, sig_cap)) .transpose()?; // The derive measured the signature entries; virtuals are measured HERE, where the target images are // loaded and a slot resolves to an address. Together they cover the whole shipped surface. let abi_imgs = load_build_images(prof, build); let core_doc: BTreeMap = serde_json::from_str(core).context("parse core gamedata for ABI shapes")?; // THE RULE FOR THIS MAP: it is keyed by NAME, and `assemble_monolith` attaches each shape to the entry // of that name — so every shape in it must have been measured at the address that entry's SHIPPED // locator resolves to. A shape measured at any other address describes a different function, and would // be published as this one's argument footprint (which is also what makes a declared prototype // checkable, so a wrong one silently converts a correct declaration into a "mismatch"). // // The Valve-table stage measured its entries as it located them, but the core-overlap removal above may // since have taken a name away from it — drop those, or the shape would outlive the locator it belongs to. let mut abi_all = valve.abi; abi_all.retain(|name, _| t3.contains_key(name)); // The derive's own measurements win on any shared name, being taken through the locator that ships. abi_all.extend(abi.iter().map(|(k, v)| (k.clone(), v.clone()))); // via the canonical inverse, not a hand-rolled key lookup: the rendered core nests its locator // (`offsets.linux`), and `entry_from_value` is the one place that shape is known. let core_offsets: Vec<(&str, String, i64)> = core_doc .iter() .filter_map(|(n, v)| { Some(( n.as_str(), class_of(n).to_string(), render::entry_from_value(v).offset?, )) }) .collect(); abi_all.extend(offset_abi_shapes( prof, &abi_imgs, core_offsets.iter().map(|(n, c, o)| (*n, c.as_str(), *o)), )); let t3_offsets: Vec<(&str, String, i64)> = t3 .iter() .filter_map(|(n, e)| Some((n.as_str(), class_of(n).to_string(), e.offset?))) .collect(); abi_all.extend(offset_abi_shapes( prof, &abi_imgs, t3_offsets.iter().map(|(n, c, o)| (*n, c.as_str(), *o)), )); // The experimental band carries its class explicitly (a guessed name need not parse as Class::Method). // A guess is measured at ITS OWN address — a different function from the one a shipped entry of the same // name locates — so names owned by `core` or by the fold are excluded, mirroring `assemble_monolith`, // which drops the guess itself for exactly this reason. This extend runs LAST, so without the guard a // guess would overwrite the shipped entry's footprint and nothing downstream would mark the swap. if let Some(gs) = experimental.as_deref() { let exp: Vec<(&str, &str, i64)> = gs .iter() .filter(|g| !g.promoted && !core_gd.contains_key(&g.name) && !t3.contains_key(&g.name)) .filter_map(|g| { Some(( g.name.as_str(), g.class.as_deref()?, g.locator.as_ref()?.offset?, )) }) .collect(); abi_all.extend(offset_abi_shapes(prof, &abi_imgs, exp.into_iter())); } // Assemble the monolith (high_confidence from the fold's own `t3`/`prov`; core/unresolved/experimental // from the derive's in-memory strings) + its CS# render, returned to `produce` — nothing written here. let (mono, cssharp) = build_monolith( prof, source_build, version, core, flagged, experimental.as_deref(), unverified, &t3, &prov, &abi_all, anchors, Some((&img, &default_lib)), )?; Ok(Folded { mono, cssharp, bindings: valve.bindings, }) } /// What the fold hands back to `produce`: the monolith, the CS# render live validation checks against, /// and the binding registry read out of the same libraries on the way past. pub(crate) struct Folded { pub mono: model::Monolith, pub cssharp: String, pub bindings: model::Bindings, } /// One experimental-band guess, built by [`emit_experimental_band`] and consumed by [`assemble_monolith`] /// into the monolith's experimental tier. Passed as a typed `Vec` in memory — NOT serialized and re-parsed, /// so `class` is a typed field a reader can see and `locator` is `None` (not a null sentinel) when the guess /// is unlocatable. `promoted` rows are skipped on read (they live in high_confidence). struct ExpGuess { name: String, tier: model::Tier, confidence: String, corroboration: String, self_named: bool, promoted: bool, collision: bool, dead_weight: bool, addr: String, /// The vtable class an OFFSET guess lives on (experimental-only eyeball aid); `None` for a sig guess. class: Option, /// The resolved locator, or `None` when the address is neither a vtable slot nor make_sig-able. locator: Option, /// The argument footprint measured at the guess's OWN address, here where that address and its image /// are both in hand. Re-deriving it later from the class name cannot reach a namespaced or templated /// class (`rtti::mangle` has no `_ZTS` needle for `Attribute::Detail::GetterT<...>`), which left 41% /// of the CS2 band shipping `abi: null` for functions whose exact address was known here. abi: Option, } /// Emit the EXPERIMENTAL band — the least-filtered inclusion tier. Every name guess in the full-slice /// universe is kept (no promote/dead-weight gate: each row is explicitly a *guess*), each attached to a /// locator derived FRESH from this binary — a vtable OFFSET where the address is a virtual slot, else a /// `make_sig` SIGNATURE — graded by [`guess_tier`], with same-name collisions flagged and cluster-sorted /// so competing guesses sit side by side. One self-describing file under a blunt UNVERIFIED banner. /// Locators are derived in parallel (the non-virtual `make_sig` majority dominates the cost). fn emit_experimental_band( prof: &GameProfile, img: &CodeImage, build: &Path, names_path: &Path, default_lib: &str, sig_cap: usize, ) -> Result> { // The name universe is either a flat [FullName] array (single-lib → the default lib) or a per-lib // { "libX.so": [FullName] } map (multilib). Either shape resolves each guess against its OWN lib image. let raw: Value = serde_json::from_str(&std::fs::read_to_string(names_path)?) .context("parse full-names json")?; let per_lib: BTreeMap> = match raw { Value::Array(_) => BTreeMap::from([( default_lib.to_string(), serde_json::from_value(raw).context("parse full-names json")?, )]), Value::Object(_) => serde_json::from_value(raw).context("parse per-lib full-names json")?, _ => { anyhow::bail!("full-names json must be a [..] array or a {{ \"lib.so\": [..] }} object") } }; // collision + protobuf clustering span the WHOLE universe (across libs): a name guessed at more than // one address — even in different libs — is flagged, and a protobuf class is seen by its full surface. let all_rows: Vec<&FullName> = per_lib.values().flatten().collect(); let mut name_count: HashMap<&str, usize> = HashMap::new(); for r in &all_rows { *name_count.entry(r.name.as_str()).or_default() += 1; } let pb_classes = protobuf_message_classes(prof, all_rows.iter().map(|r| r.name.as_str())); // resolve each lib's guesses against that lib's OWN image — the default lib reuses the already-loaded // `img`; the others load on demand (a guess for a lib absent from this build is skipped). make_sig (the // non-virtual majority) dominates the cost, so each lib's batch runs in parallel. let mut extra_imgs: HashMap = HashMap::new(); let mut entries: Vec<(u8, bool, String, String, ExpGuess)> = Vec::new(); let mut skipped_libs: Vec = Vec::new(); for (lib_file, rows) in &per_lib { let libname = lib_name_from_file(lib_file); let limg: &CodeImage = if libname == default_lib { img } else { if !extra_imgs.contains_key(lib_file) { let Ok(loaded) = load_lib(build, lib_file) else { // a whole library's guesses drop out — report it rather than shrinking the band silently skipped_libs.push(format!("{lib_file} ({})", rows.len())); continue; }; extra_imgs.insert(lib_file.clone(), loaded); } &extra_imgs[lib_file] }; // addr -> (class, slot): the first primary vtable holding the address == a directly callable offset. let mut vt_of: HashMap = HashMap::new(); for cv in rtti::enumerate_vtables(limg, prof.max_vtable_slots) .into_iter() .filter(|c| c.offset_to_top == 0) { for (i, &a) in cv.slots.iter().enumerate() { vt_of.entry(a).or_insert_with(|| (cv.name.clone(), i)); } } let idxs: Vec = (0..rows.len()).collect(); let lib_entries: Vec<(u8, bool, String, String, ExpGuess)> = parallel_map(&idxs, default_threads(None), |&i| { let r = &rows[i]; let (rank, tier) = guess_tier(r); let collision = name_count.get(r.name.as_str()).copied().unwrap_or(0) > 1; // protobuf / foreign-namespace / serializer plumbing: kept (nothing hidden) but FLAGGED, so // a reader can grep it out and the confidence tiers aren't polluted by generic-method matches // (e.g. a protobuf `::Clear` corroborating as "exact" on the dictionary word). Also sunk to // the bottom of its tier via the sort key below, so real game functions surface first. let dead = is_dead_weight_name(prof, &r.name) || is_serializer_plumbing(prof, &r.name, &pb_classes); let addr = parse_hex_addr(&r.addr); let abi = addr.and_then(|a| abi::abi_shape(limg, a)).map(shipped_abi); let (class, locator): (Option, Option) = match addr.and_then(|a| vt_of.get(&a).map(|v| (a, v))) { // a vtable slot: the canonical offset locator + the experimental-only `class` tag // (the vtable it lives on) so a reader can eyeball an offset guess. Some((_, (cls, slot))) => ( Some(cls.clone()), Some(model::Entry { offset: Some(*slot as i64), ..Default::default() }), ), None => match addr.and_then(|a| emit::make_sig(limg, a, sig_cap)) { Some(sig) => ( None, Some(model::Entry { signature: Some(model::Sig { library: libname.clone(), linux: sig, }), ..Default::default() }), ), None => (None, None), }, }; let entry = ExpGuess { name: r.name.clone(), tier, confidence: r.confidence.clone(), corroboration: r.corroboration.clone(), self_named: r.self_named, promoted: r.promote, collision, dead_weight: dead, addr: r.addr.clone(), class, locator, abi, }; (rank, dead, r.name.clone(), r.addr.clone(), entry) }); entries.extend(lib_entries); } // cluster-sort: tier first (the confidence grouping), then dead-weight LAST within a tier (real game // functions surface above the plumbing), then name (collisions become adjacent), then addr. entries.sort_by(|a, b| (a.0, a.1, &a.2, &a.3).cmp(&(b.0, b.1, &b.2, &b.3))); let (mut collisions, mut unlocatable, mut dead_weight) = (0usize, 0usize, 0usize); let guesses: Vec = entries.into_iter().map(|(_, _, _, _, e)| e).collect(); for e in &guesses { collisions += usize::from(e.collision); dead_weight += usize::from(e.dead_weight); unlocatable += usize::from(e.locator.is_none()); } let total = guesses.len(); eprintln!( "experimental band: {total} guesses ({collisions} name-collisions, {dead_weight} dead-weight-flagged, \ {unlocatable} unlocatable)" ); if !skipped_libs.is_empty() { eprintln!( " libs absent from this build (their guesses dropped): {}", skipped_libs.join(", ") ); } Ok(guesses) } // ══════════════════════════════════════════════════════════════════════════════════════════════ // §5 · MONOLITH ASSEMBLY // ══════════════════════════════════════════════════════════════════════════════════════════════ pub(crate) type GdMap = serde_json::Map; // A gamedata entry's locator (a byte SIGNATURE or a vtable OFFSET) is `model::Entry`: construct one with // `Entry::signature`/`Entry::offset`, render it with `render::locator_value`, parse it back with // `render::entry_from_value` — one type, one write/read pair in `core`, no separate enum or ad-hoc `.get()`. /// Parse a gamedata json that MAY carry `//` line-comment banners, from an in-memory string — the /// disk-free path the fold takes for the derive's rendered core and the live validate takes for the /// monolith's CS# render, both avoiding a disk round-trip. pub(crate) fn read_gamedata_str(raw: &str) -> Result { let stripped = raw .lines() .filter(|l| !l.trim_start().starts_with("//")) .collect::>() .join("\n"); serde_json::from_str(&stripped).context("parse gamedata json") } /// Assemble the monolith (the tiered catalogue that becomes `rosetta-.json`'s `functions`) from /// the in-memory derive/fold results: /// the rendered `core_json` → core tier; the fold's `t3`/`prov` → high_confidence; the `experimental` guesses /// → the non-promoted experimental tail; the `flagged` list → unresolved. Pure assembly, no re-derivation and /// no intermediate files. `validated` stays `None` (the live stage annotates it later). /// Experimental is name-keyed: a collision (one name guessed at several addresses, already best-first in the /// input order) collapses to its best representative with `collision:true` retained. fn assemble_monolith( prof: &GameProfile, source_build: &str, version: &str, core_json: &str, flagged: &[model::Flagged], experimental: Option<&[ExpGuess]>, unverified: &BTreeSet, t3: &BTreeMap, prov: &BTreeMap, abi: &BTreeMap, ) -> Result { use model::{Counts, MonoEntry, MonoMeta, Monolith, Provenance, Tier, Unresolved}; // core tier: every guaranteed entry in the derive's rendered core, minimal provenance. // An entry whose fingerprint check rejected every candidate (emitted only because ONE era-sig resolved // uniquely) is marked `catalogue-unverified` — it is in `core` on locator strength alone, and nothing // downstream, live validation included, re-checks that it is the right function. let mut core: BTreeMap = BTreeMap::new(); { // ABI prototype-drift detail per name, from the flag list — surfaced onto the core entry (the sig // still ships; `assemble` would otherwise drop the abi-drift flag since its name is already in core). let abi_drift: BTreeMap<&str, &str> = flagged .iter() .filter(|f| f.reason == model::FlagReason::AbiDrift) .map(|f| (f.name.as_str(), f.detail.as_str())) .collect(); let doc: BTreeMap = serde_json::from_str(core_json).context("parse core gamedata")?; for (name, v) in &doc { let source = if unverified.contains(name) { "catalogue-unverified" } else { "catalogue" }; core.insert( name.clone(), MonoEntry { locator: render::entry_from_value(v), abi: abi.get(name.as_str()).cloned(), provenance: Provenance { source: Some(source.into()), abi_drift: abi_drift.get(name.as_str()).map(|d| d.to_string()), ..Provenance::with_tier(Tier::Core) }, validated: None, aliases: Vec::new(), // filled once both shipped tiers exist }, ); } } // high_confidence tier: the promoted names — provenance + locator taken from the IN-MEMORY fold (`t3` = // name -> typed `Entry`, `prov` = name -> typed provenance), so no disk round-trip is needed to build the // monolith and no untyped re-parse stands between fold and shipped model. let mut high: BTreeMap = BTreeMap::new(); for (name, provenance) in prov { let locator = t3.get(name).cloned().unwrap_or_default(); high.insert( name.clone(), MonoEntry { locator, abi: abi.get(name.as_str()).cloned(), provenance: provenance.clone(), validated: None, aliases: Vec::new(), // filled once both shipped tiers exist }, ); } // experimental tier: the graded guess tail, EXCLUDING promoted ones (they are high_confidence) and any // name already owned by core/high_confidence. First occurrence wins per name (best-first order). Read // straight off the typed `Vec` — no serialize-and-re-parse, so a renamed field is a compile // error here rather than a silent `None`. let mut exp_tier: BTreeMap = BTreeMap::new(); if let Some(guesses) = experimental { for g in guesses { if g.promoted { continue; // promoted → it lives in high_confidence } if core.contains_key(&g.name) || high.contains_key(&g.name) { continue; // a higher tier already owns this name } let Some(locator) = &g.locator else { continue; // unlocatable guess — not shippable }; // The experimental tier's provenance (the rest of the fields stay None). let provenance = Provenance { addr: Some(g.addr.clone()), confidence: Some(g.confidence.clone()), self_named: Some(g.self_named), corroboration: Some(g.corroboration.clone()), collision: Some(g.collision), dead_weight: Some(g.dead_weight), ..Provenance::with_tier(g.tier) }; exp_tier.entry(g.name.clone()).or_insert(MonoEntry { // The class rides on the LOCATOR now, not beside it: for an offset entry the class is what // makes the slot index mean anything, and keeping them together is what lets it reach the // emitters through `Monolith::select`. locator: model::Entry { class: g.class.clone(), ..locator.clone() }, // The band's own measurement first: it was taken at the guess's exact address, which IS // what the locator resolves to. The name-keyed map is the fallback. abi: g.abi.clone().or_else(|| abi.get(g.name.as_str()).cloned()), provenance, validated: None, // Never grouped: an unverified name sharing a target with another unverified name is not // evidence they mean the same thing. This band states the converse through // `provenance.collision` (one name guessed at several addresses). aliases: Vec::new(), }); } } // unresolved tier: catalogue functions the derivation couldn't produce. let mut unresolved: BTreeMap = BTreeMap::new(); for f in flagged { // A flagged function can still have been resolved by another path (e.g. a sig-drifted entry // recovered as a vtable offset lands in `core`). `unresolved` means truly no locator in any // tier, so skip anything a resolved tier already owns — this keeps the four tiers disjoint. if core.contains_key(&f.name) || high.contains_key(&f.name) || exp_tier.contains_key(&f.name) { continue; } // One name can be flagged by more than one stage (a catalogue entry carrying both a sig and a // vtable-offset variant fails each independently). The tier is name-keyed so only one reason // fits: keep the FIRST, since the list runs in derivation order and the earlier stage's reason // is the more specific one. unresolved .entry(f.name.clone()) .or_insert_with(|| Unresolved { reason: f.reason.as_str().to_string(), detail: f.detail.clone(), }); } let (alias_groups, aliased_names) = link_aliases(&mut core, &mut high); let counts = Counts { core: core.len(), high_confidence: high.len(), experimental: exp_tier.len(), unresolved: unresolved.len(), }; Ok(Monolith { meta: MonoMeta { game_key: prof.game_key.to_string(), game: prof.display_name.to_string(), source_build: source_build.to_string(), version: version.to_string(), counts, alias_groups, aliased_names, }, core, high_confidence: high, experimental: exp_tier, unresolved, }) } /// Cross-link the shipped tiers' ALIASES: names that locate the same function, written onto each entry as /// the other names for it. Returns `(groups, names covered)` for the meta counters. /// /// **Locator identity is the key, and for a signature it is exactly address identity.** A shipped pattern is /// generated AT the resolved address and then confirmed unique within its library, so two entries carrying /// the same `(library, pattern)` cannot resolve anywhere but the same single address — the uniqueness check /// the emitter already performs is what makes string equality a sound proxy here, with no second scan. A /// vtable entry keys on `(class, slot)`, which is the same argument: the slot means nothing except relative /// to a named vtable, and together they name one function. /// /// **A bare slot — an `offset` with no `class` — is deliberately left ungrouped.** It names no vtable, so two /// of them sharing a slot index are not evidence of anything; grouping them anyway put 1,080 CS2 names into /// 70 fictitious groups when measured. Those entries ship with no `aliases` because none is derivable, not /// because none exists, and the field's doc says so rather than letting a reader infer uniqueness from the /// silence. fn link_aliases( core: &mut BTreeMap, high: &mut BTreeMap, ) -> (usize, usize) { // The locator's identity as a comparable key, or None where the entry locates nothing groupable. let key = |e: &model::MonoEntry| -> Option<(String, String)> { if let Some(s) = &e.locator.signature { return Some((s.library.clone(), s.linux.clone())); } match (&e.locator.class, e.locator.offset) { (Some(c), Some(slot)) => Some((c.clone(), format!("#{slot}"))), _ => None, } }; // Group ACROSS the two tiers, not within each: a `core` name and a `high_confidence` name on one address // are aliases, and that pairing is the one a consumer is least likely to spot unaided. A `BTreeSet` per // target rather than a `Vec` because the tiers are only disjoint by construction, and a name reaching // this twice must not inflate its own group or turn up as its own alias. let mut by_target: BTreeMap<(String, String), BTreeSet> = BTreeMap::new(); for (name, e) in core.iter().chain(high.iter()) { if let Some(k) = key(e) { by_target.entry(k).or_default().insert(name.clone()); } } by_target.retain(|_, names| names.len() > 1); // Resolve every membership first, then write: an entry is reached through whichever tier holds it, // and both are written the same way rather than one path shadowing the other. let mut others_of: BTreeMap> = BTreeMap::new(); for names in by_target.values() { for name in names { others_of.insert( name.clone(), names.iter().filter(|n| *n != name).cloned().collect(), ); } } for (name, e) in core.iter_mut().chain(high.iter_mut()) { if let Some(others) = others_of.get(name) { e.aliases = others.clone(); } } (by_target.len(), others_of.len()) } /// Fold the live-validation verdict INTO the monolith (Stage F), from the `verdicts` map the live oracle /// returns. /// /// **THREE-valued, and that is the whole point of the function.** `Some(true)` = the running server /// confirmed it; `Some(false)` = the oracle dropped it confident-bad; `None` = the oracle could not check /// it at all (a library this run did not map, a non-vtable class), and the artifact must say so rather /// than pick a side. A two-state reading — "present means good, absent means bad" — is exactly the /// collapse this exists to prevent, because it would report an unchecked entry as validated. /// /// Only the shipped tiers (`core` + `high_confidence`) are annotated; `experimental` was never sent to the /// server, so it keeps `None`. Keeps the monolith self-contained — no standalone validation sidecar. pub(crate) fn annotate_validation( mono: &mut model::Monolith, verdicts: &BTreeMap>, ) { // `verdicts` carries the HONEST per-entry outcome from the live oracle: Some(true) = live-confirmed, // None = kept but the oracle could NOT check it (library not mapped this run, non-vtable class, or an // out-of-bounds-but-kept offset that was demoted to Some(false)) — so the artifact must not claim // validation; an entry ABSENT from `verdicts` was dropped by the oracle -> Some(false). The monolith must // never assert a live validation it did not receive ("never lies"). let annotate = |m: &mut BTreeMap| { for (name, e) in m.iter_mut() { e.validated = verdicts.get(name).copied().unwrap_or(Some(false)); } }; annotate(&mut mono.core); annotate(&mut mono.high_confidence); let all = || mono.core.values().chain(mono.high_confidence.values()); let validated = all().filter(|e| e.validated == Some(true)).count(); let unverified = all().filter(|e| e.validated.is_none()).count(); let failed = all().filter(|e| e.validated == Some(false)).count(); eprintln!( " monolith validation annotated: {validated} live-validated, {unverified} kept-unverified, \ {failed} failed (core + high_confidence)" ); } // ══════════════════════════════════════════════════════════════════════════════════════════════ // §6 · CORPUS-MODEL DISTILLATION // ══════════════════════════════════════════════════════════════════════════════════════════════ /// A vtable-offset function's class + its dated offset history (from gamedata across eras). struct VtFunc { name: String, class: String, timeline: Vec<(String, i64)>, // (date, offset), sorted ascending by date } fn parse_offset(v: &str) -> Option { v.rsplit('=').next().unwrap_or(v).trim().parse().ok() // handles "102" and "linux=102" } /// Build each catalogue function's dated vtable-offset timeline: sorted `(YYYY-MM-DD, offset)` pairs, /// skipping functions with no dated linux/any vtable-offset variant. The shared input to both the model /// distiller's slot chaining and gamedata's offset derivation — each offset variant's `src` carries its /// own date (baked into the catalogue), so there is no external tag→date map. fn vtable_offset_timelines(cat: &[Func]) -> Vec { let mut vfuncs: Vec = Vec::new(); for f in cat { let mut tl: Vec<(String, i64)> = Vec::new(); for v in &f.variants { if v.kind != VariantKind::VtableOffset || !v.platform.is_linux() { continue; } // `src` is the offset's `YYYY-MM-DD` commit/build date. let d = (v.src.len() >= 10 && v.src.as_bytes().get(4) == Some(&b'-')) .then(|| v.src.clone()); if let (Some(d), Some(o)) = (d, parse_offset(&v.value)) { tl.push((d, o)); } } if tl.is_empty() { continue; } tl.sort(); vfuncs.push(VtFunc { name: f.name.clone(), class: class_of(&f.name).to_string(), timeline: tl, }); } vfuncs } /// Fingerprint every slot address of a vtable: `slot index -> its structural fingerprint` (`None` where /// a slot doesn't decode). The per-class vtable signature the cross-build slot alignment operates on. fn slot_fingerprints(img: &CodeImage, slots: &[u64]) -> Vec>> { slots .iter() .map(|&a| fingerprint::extract(img, a).map(|f| f.to_vec())) .collect() } /// As [`slot_fingerprints`], but MEMOIZES `fingerprint::extract` per function address — a single inherited /// method (a `CBaseEntity` slot) appears at the same address in hundreds of subclasses' vtables, so the /// distill re-fingerprints it once per owning class without this. `extract` is pure/deterministic, so the /// memo returns byte-identical vectors. CAVEAT: the memo MUST be scoped to ONE image — the offline /// server/engine2 `CodeImage`s share low vaddr ranges, so a cross-image bare-vaddr memo would be a /// correctness/output bug. `extract_build_vtables` creates a fresh memo per image for exactly this reason. fn slot_fingerprints_memo( img: &CodeImage, slots: &[u64], memo: &mut HashMap>>, ) -> Vec>> { slots .iter() .map(|&a| { memo.entry(a) .or_insert_with(|| fingerprint::extract(img, a).map(|f| f.to_vec())) .clone() }) .collect() } pub(crate) fn build_date(label: &str) -> String { label.chars().take(10).collect() // "2024-08-21_231850" -> "2024-08-21" } /// The offset in effect at `date` = the latest era observation with date <= `date`. fn offset_at(timeline: &[(String, i64)], date: &str) -> Option { timeline .iter() .rfind(|(d, _)| d.as_str() <= date) .map(|(_, o)| *o) } /// One build's identity in the corpus model: its directory label and derived date. #[derive(serde::Serialize, serde::Deserialize)] struct BuildMeta { label: String, date: String, } /// A function's distilled ABI shape — the argument footprint + return class, serializable for the model. /// The consensus of the corpus's newest builds; a target whose derived shape differs is a prototype /// drift the byte-signature can't see, flaggable at DERIVE time without the raw corpus. #[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] struct AbiSig { int: u8, float: u8, ret: abi::RetClass, // typed; serializes to the describe() token ("ret=int", …) } impl From for AbiSig { fn from(s: abi::AbiShape) -> Self { AbiSig { int: s.int_args, float: s.float_args, ret: s.ret_class, } } } impl AbiSig { /// True when this shape DEFINITELY differs from `other`: the argument footprint moved, or both /// return classes are definite (not `Unknown`) and differ. An `Unknown` return is "no signal" and /// never raises a drift flag on its own. fn differs(&self, other: &AbiSig) -> bool { self.int != other.int || self.float != other.float || (self.ret != other.ret && self.ret != abi::RetClass::Unknown && other.ret != abi::RetClass::Unknown) } fn brief(&self) -> String { format!( "int={} float={} {}", self.int, self.float, self.ret.describe() ) } } /// One reference build's per-sig-fn observations: `(name, fingerprint, optional ABI shape)`. type RefObs = Vec<(String, Vec, Option)>; /// The model's `consensus_abi` field from its raw `abi_obs` windows — the modal ABI shape per sig fn. /// SHARED by the full distill and the incremental fold (like `resolved_slot_of`/`extract_build_vtables`): /// `consensus_abi` is a `CorpusModel` field and the fold contracts byte-equality with a re-distill, so an /// inline copy edited in one path but not the other would break that silently. fn consensus_from_windows( abi_obs: &BTreeMap>, ) -> BTreeMap { abi_obs .iter() .filter_map(|(n, obs)| { mode_abi(obs.iter().map(|(_, s)| s.clone()).collect()).map(|s| (n.clone(), s)) }) .collect() } /// The modal shape across a function's per-build observations — robust to an occasional decode miss. /// Ties break to the larger shape by `Ord`, so the consensus is deterministic. fn mode_abi(mut obs: Vec) -> Option { obs.sort(); let mut best: Option<(AbiSig, usize)> = None; let mut i = 0; while i < obs.len() { let run = 1 + obs[i + 1..].iter().take_while(|x| **x == obs[i]).count(); if best.as_ref().is_none_or(|(_, c)| run >= *c) { best = Some((obs[i].clone(), run)); } i += run; } best.map(|(s, _)| s) } /// A vtable slot index stored in 4 bytes instead of `Option`'s 16. Slot indices max in the low /// hundreds, so a `u32` with `u32::MAX` reserved for "no match" is lossless — and it quarters the RAM of /// the model's `hops`/`resolved_slot`, which are its dominant cost on a long-history, many-class corpus /// (Dota's clean model is ~10x CS2's in both class-slots and builds; the `Option` hops alone would /// be tens of GB). Serializes to the SAME JSON as `Option` — a bare number, or `null` for none — /// so the model file format and every previously-produced model are unchanged; the win is purely /// build-time RAM and on-disk size. #[derive(Clone, Copy, PartialEq, Eq)] struct Slot(u32); impl Slot { const NONE: Slot = Slot(u32::MAX); fn from_opt(o: Option) -> Slot { o.map_or(Slot::NONE, |v| Slot(v as u32)) } fn get(self) -> Option { (self.0 != u32::MAX).then_some(self.0 as usize) } } impl serde::Serialize for Slot { fn serialize(&self, s: S) -> Result { match self.get() { Some(v) => s.serialize_u64(v as u64), None => s.serialize_none(), } } } impl<'de> serde::Deserialize<'de> for Slot { fn deserialize>(d: D) -> Result { Ok(Slot::from_opt( Option::::deserialize(d)?.map(|v| v as usize), )) } } /// One consecutive-build vtable alignment (old slot index -> new slot). The ~93 % of alignments that are pure /// IDENTITY (slot `i` still maps to slot `i`, no drops) — plus the empty alignment — are carried as just their /// LENGTH, both on disk (a single integer, not the full `[0,1,2,…,n-1]` array) AND IN MEMORY (the `Identity` /// variant, not a materialized `Vec`). `hops` is the model's dominant field at scale (per-class × /// per-build-PAIR, so it grows with corpus length), and it is overwhelmingly identity, so this collapses the /// on-disk size, the JSON parse cost, AND the resident RAM + per-load allocations (a model load does not /// materialize millions of small identity `Vec`s) — with no derivation change. A shifted alignment keeps the /// explicit slot list (`Explicit`, `null` for a dropped slot). `deserialize` accepts BOTH forms, so a full /// identity array also loads and normalizes to `Identity`. #[derive(Clone, PartialEq, Eq)] enum Hop { /// identity/empty alignment of this length: slot `i` maps to `i`. The compressible common case. Identity(u32), /// a shifted alignment: the explicit old-slot -> new-slot list (`Slot::NONE` for a dropped slot). Explicit(Vec), } impl Default for Hop { fn default() -> Hop { Hop::Identity(0) // the empty alignment } } impl Hop { /// Classify a raw alignment into the compact form: a pure identity/empty run (every present slot maps to /// its own index) collapses to `Identity(len)`, anything shifted stays `Explicit`. fn from_slots(v: Vec) -> Hop { if v.iter().enumerate().all(|(i, s)| s.get() == Some(i)) { Hop::Identity(v.len() as u32) } else { Hop::Explicit(v) } } /// The alignment as one `Option` per old slot — the form the derive/backfill readers consume /// (identity materializes `(0..len)` on demand rather than storing it). fn to_options(&self) -> Vec> { match self { Hop::Identity(n) => (0..*n as usize).map(Some).collect(), Hop::Explicit(v) => v.iter().map(|s| s.get()).collect(), } } } impl serde::Serialize for Hop { fn serialize(&self, s: S) -> Result { match self { Hop::Identity(n) => s.serialize_u64(*n as u64), // identity (incl. empty) -> its length Hop::Explicit(v) => v.serialize(s), // shifted -> the explicit slot array } } } impl<'de> serde::Deserialize<'de> for Hop { fn deserialize>(d: D) -> Result { use serde::de::{self, SeqAccess, Visitor}; struct HopVisitor; impl<'de> Visitor<'de> for HopVisitor { type Value = Hop; fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { f.write_str("an integer (identity-alignment length) or an array of slot indices") } // integer form: an identity alignment of this length -> slot i maps to i. fn visit_u64(self, v: u64) -> Result { Ok(Hop::Identity(v as u32)) } fn visit_i64(self, v: i64) -> Result { self.visit_u64(v as u64) } // array form: normalize through `from_slots` so a pre-compression identity array collapses to // `Identity` (identical to the integer form), a shift stays `Explicit`. fn visit_seq>(self, mut seq: A) -> Result { let mut v = Vec::new(); while let Some(s) = seq.next_element::()? { v.push(s); } Ok(Hop::from_slots(v)) } } d.deserialize_any(HopVisitor) } } /// Distilled cross-build corpus signals — everything `gamedata` reads from the full build corpus, /// WITHOUT the Valve binaries. Fingerprints/offsets/slot-indices are derived facts (zero /// Valve bytes) → shippable. Produced by `corpus-model`; consumed by `produce --corpus-model` /// alongside only the target build's binary. Incremental: model N + build N+1 → model N+1. #[derive(serde::Serialize, serde::Deserialize)] pub struct CorpusModel { /// The class scope this model was distilled with. The incremental fold recomputes the tracked-class set /// with the CLI's `--class-scope`, so a fold whose scope differs from the distill's would silently /// produce a model that is NOT the re-distill it claims to be — the fold `ensure!`s they match. Defaults /// to `Clean` (what every production model uses) when the field is absent. #[serde(default)] class_scope: ClassScope, builds: Vec, // chronological /// class -> per consecutive build-pair: the vtable slot alignment (old slot -> new slot). Each alignment /// is a [`Hop`], which serializes the identity-alignment common case as a single length integer. hops: BTreeMap>, /// class -> per build: vtable slot count (offset in-bounds checks). slot_counts: BTreeMap>, /// sig function -> per build: the vtable slot its signature resolved to (virtual-anchor timeline). resolved_slot: BTreeMap>, /// sig function -> `(build index, raw fingerprint)` across the newest `CORPUS_REF_K` BUILDS, newest /// first — the sig-hit verification set. /// /// The build index is what makes the window well-defined. Without it the incremental fold could only /// keep "the newest K OBSERVATIONS", which for a function that fails to resolve in some builds retains /// fingerprints from arbitrarily far back — a superset of what a re-distill produces, drifting further /// with every fold, and one the fold==re-distill equivalence claim could never actually hold against. /// With it, the fold evicts by build AGE and reproduces the re-distill's window exactly. ref_fps: BTreeMap)>>, /// latest build's per-class vtable slot fingerprints — to align the NEXT build without its binary. /// (BTreeMap like the other fields, so the serialized model is deterministic; the transient per-build /// `VtableFps` stays a HashMap — this is the one boundary where they convert.) latest_vtable_fps: BTreeMap>>>, /// sig function -> its consensus ABI shape across the newest builds. Lets `produce --corpus-model` /// flag a target whose derived shape differs from history at DERIVE time — the prototype-drift signal /// `abi-diff` gives, but forward and corpus-free. consensus_abi: BTreeMap, /// sig function -> `(build index, raw ABI shape)` over the newest builds, mirroring [`Self::ref_fps`]. /// The window `consensus_abi` is the mode of — stored so an incremental fold can re-window it and /// recompute the consensus byte-exactly rather than approximate from the collapsed mode. abi_obs: BTreeMap>, } /// Number of newest builds whose raw sig fingerprints go into the model's verification set. const CORPUS_REF_K: usize = 8; /// The per-build array shape every downstream reader assumes. /// /// `slot_counts`, `hops` and `resolved_slot` are read by build INDEX, so a length desync does not fail — /// it silently re-dates an observation, attributing this build's slot to some older one. Checked where the /// model is WRITTEN rather than only where it is rolled forward, because a distill that emits a malformed /// model is the same defect one fold later and the fold's error would then name the wrong producer. /// O(classes) against a pass that just walked whole binaries, so it runs on the production path rather /// than behind a debug assert. fn check_model_shape(m: &CorpusModel) -> Result<()> { let nb = m.builds.len(); for (c, v) in &m.slot_counts { ensure!( v.len() == nb, "slot_counts[{c}] has {} entries, expected {nb}", v.len() ); } for (c, v) in &m.hops { ensure!( v.len() == nb - 1, "hops[{c}] has {} entries, expected {}", v.len(), nb - 1 ); } for (n, v) in &m.resolved_slot { ensure!( v.len() == nb, "resolved_slot[{n}] has {} entries, expected {nb}", v.len() ); } Ok(()) } /// The set of classes a model tracks hops for — THE rule that has to agree between the full distill and the /// incremental fold, since `fold_model_cmd` claims to produce a model equal to a re-distill over the same /// builds and CI depends on that. Shared for the same reason `extract_build_vtables` and `ref_obs_of_build` /// are: a divergence here would be silent, and the fold is the standing production path. /// /// `clean` (default) = every REAL game class in the latest build — enough for any modding/PR offset to /// derive model-only, without the dead weight (template stamps / protobuf / NetworkVar chainers whose hops /// nobody derives an offset from, and whose ML "patterns" live in the fingerprint matrix, not here). `all` /// keeps those too; `catalogue` keeps only what the catalogue names. The catalogue's own classes are ALWAYS /// unioned in (the derivation needs them). CI compresses the model on release, so the uncompressed on-disk /// size is not the shipped size. fn tracked_classes( prof: &GameProfile, class_scope: ClassScope, latest_imgs: &[CodeImage], vfuncs: &[VtFunc], sig_funcs: &[&Func], ) -> Vec { let class_ok = |name: &str| match class_scope { ClassScope::All => true, ClassScope::Catalogue => false, ClassScope::Clean => { !is_dead_weight_class(prof, name) && !name.contains('<') && !name.contains("NetworkVar_") } }; latest_imgs .iter() .flat_map(|img| rtti::enumerate_vtables(img, prof.max_vtable_slots)) .filter(|c| c.offset_to_top == 0) .map(|c| c.name) .filter(|n| class_ok(n)) .chain(vfuncs.iter().map(|f| f.class.clone())) .chain(sig_funcs.iter().map(|f| class_of(&f.name).to_string())) .collect::>() .into_iter() .collect() } pub fn corpus_model_cmd( prof: &GameProfile, catalogue: &Path, corpus: &Path, class_scope: ClassScope, out: &Path, ) -> Result<()> { let cat = load_catalogue(catalogue)?; // dated vtable-offset timelines -> the classes we chain (same as gamedata). let vfuncs = vtable_offset_timelines(&cat); let sig_funcs: Vec<&Func> = cat.iter().filter(|f| !linux_sigs(f).is_empty()).collect(); let mut builds: Vec<(String, PathBuf)> = find_builds(prof, corpus)? .into_iter() .map(|p| (build_date(&label_of(&p)), p)) .collect(); builds.sort(); ensure!(!builds.is_empty(), "no builds under {}", corpus.display()); // SCOPED so the newest build's images are freed before the streaming loop starts. Holding them for // the whole run would pin one extra complete image set — file bytes plus each image's relocation and // symbol maps — against this command's whole point, which is bounded RAM over ~1k builds. let classes = { let latest_imgs = load_build_images(prof, &builds.last().unwrap().1); tracked_classes(prof, class_scope, &latest_imgs, &vfuncs, &sig_funcs) }; eprintln!( "distilling {} builds x {} classes (scope={class_scope:?}; reads the corpus once) ...", builds.len(), classes.len() ); // The one expensive pass over the binaries: each class's vtable slot fingerprints+addresses and each sig // function's resolved address, per build. Builds are processed ONE AT A TIME in chronological order — the // parallelism lives INSIDE each build's extract (the `resolve_unique` sweep is ~98% of the cost and fans // out across all cores; see `extract_build_vtables`), so a single build saturates the machine while only // that build's fps (plus the previous build's, for the next hop) are ever resident. That keeps peak RAM at // ~1.5 builds — which is what makes a deep game (Dota's fat fps) // fit. The fat per-slot fingerprints are the memory hog; the distilled signals below consume them only // pairwise (nw_align of consecutive builds) plus the final build's set, so we carry only the previous // build's fps across the loop and drop the rest. Byte-identical to a whole-corpus distill (same // consecutive pairs, same order). let want: HashSet<&str> = classes.iter().map(String::as_str).collect(); let nb = builds.len(); let mut hop_acc: HashMap>> = classes .iter() .map(|c| (c.clone(), Vec::with_capacity(nb.saturating_sub(1)))) .collect(); let mut count_acc: HashMap> = classes .iter() .map(|c| (c.clone(), Vec::with_capacity(nb))) .collect(); let mut rslot_acc: HashMap> = sig_funcs .iter() .map(|f| (f.name.clone(), Vec::with_capacity(nb))) .collect(); let mut prev_fps: Option = None; let nthreads = default_threads(None); for (_, dir) in &builds { let bv = extract_build_vtables(prof, dir, &want, &sig_funcs, nthreads); // A build whose libs are present-but-unparseable poisons every chain that crosses it (empty // fingerprints ⇒ empty hops ⇒ an unconditional chain break), and `find_builds` admits builds on // file existence alone — so say so instead of folding the damage in silently. if bv.empty { bail!( "build {} loaded ZERO libraries — it would contribute empty hops and sever every \ anchor chain crossing it; remove or repair it, then re-distill", bv.date ); } if !bv.unreadable.is_empty() { eprintln!( " WARNING build {}: {} unparseable ({}) — its fingerprints are partial", bv.date, bv.unreadable.len(), bv.unreadable.join(", ") ); } // Per-class facts — the slot count and the vtable-slot hop against the previous build — computed in // PARALLEL over the class set. `parallel_map` returns results in input (class) order, so pushing them // into the by-name accumulators below is byte-identical to a serial per-class loop — // the build loop itself stays serial (chronology is the hop's `from`, and each key still gets one push // per build). count is folded into the same pass (trivial, but it spares a second walk of the classes). let per_class: Vec<(usize, Option>)> = match &prev_fps { Some(prev) => parallel_map(&classes, nthreads, |c| { let count = bv.fps.get(c).map_or(0, Vec::len); // hop empty where either side lacks the class — identical to the windows(2) pairing of a // whole-corpus distill. let hv: Vec = match (prev.get(c), bv.fps.get(c)) { (Some(a), Some(b)) => nw_align(a, b).into_iter().map(Slot::from_opt).collect(), _ => Vec::new(), }; (count, Some(hv)) }), // first build: a count for every class, but no hop yet (nothing to align against). None => parallel_map(&classes, nthreads, |c| { (bv.fps.get(c).map_or(0, Vec::len), None) }), }; for (c, (count, hop)) in classes.iter().zip(per_class) { count_acc.get_mut(c).unwrap().push(count); if let Some(hv) = hop { hop_acc.get_mut(c).unwrap().push(hv); } } // the vtable slot each sig fn resolved to this build (position of its resolved addr in the vtable), // likewise parallel over the sig set and reassembled in input order. let rslots: Vec = parallel_map(&sig_funcs, nthreads, |f| { Slot::from_opt(resolved_slot_of(&bv, f)) }); for (f, s) in sig_funcs.iter().zip(rslots) { rslot_acc.get_mut(&f.name).unwrap().push(s); } prev_fps = Some(bv.fps); // carry only this build's fps: next hop's `from`, and at the end `latest` } // distill -> the small, shippable signals (assembled in classes / sig_funcs key order) let hops: BTreeMap> = classes .iter() .map(|c| { ( c.clone(), hop_acc .remove(c) .unwrap_or_default() .into_iter() .map(Hop::from_slots) .collect(), ) }) .collect(); let slot_counts: BTreeMap> = classes .iter() .map(|c| (c.clone(), count_acc.remove(c).unwrap_or_default())) .collect(); let resolved_slot: BTreeMap> = sig_funcs .iter() .map(|f| { ( f.name.clone(), rslot_acc.remove(&f.name).unwrap_or_default(), ) }) .collect(); // convert the transient (HashMap) newest-build fps into the model's deterministic BTreeMap field. let latest_vtable_fps: BTreeMap>>> = prev_fps.unwrap_or_default().into_iter().collect(); // reference fingerprints of each sig fn in the newest CORPUS_REF_K builds (verification set, raw) // newest-first, carrying each build's ABSOLUTE index so the window is defined by build age (see // `CorpusModel::ref_fps`) rather than by how many observations happened to succeed. let ref_dirs: Vec<(u32, PathBuf)> = builds .iter() .enumerate() .rev() .take(CORPUS_REF_K) .map(|(i, (_, p))| (i as u32, p.clone())) .collect(); // SERIAL over the (few) ref builds so `ref_obs_of_build`'s inner sig fan-out (nthreads-wide) drives the // cores directly — an outer parallel_map here would nest and oversubscribe. Order is preserved either way // (results keyed by build index + name downstream), so this is byte-identical. let per_ref: Vec<(u32, RefObs)> = ref_dirs .iter() .map(|(i, dir)| (*i, ref_obs_of_build(prof, dir, &cat, &sig_funcs, nthreads))) .collect(); let mut ref_fps: BTreeMap)>> = BTreeMap::new(); let mut abi_obs: BTreeMap> = BTreeMap::new(); for (idx, v) in per_ref { for (n, fp, sh) in v { if let Some(s) = sh { abi_obs.entry(n.clone()).or_default().push((idx, s)); } ref_fps.entry(n).or_default().push((idx, fp)); } } // consensus ABI shape per sig fn = the modal shape over the newest builds (derive-time drift check). // Computed by reference so the raw `abi_obs` window survives into the model (the incremental fold needs it). let consensus_abi = consensus_from_windows(&abi_obs); let model = CorpusModel { class_scope, builds: builds .iter() .map(|(d, p)| BuildMeta { label: label_of(p), date: d.clone(), }) .collect(), hops, slot_counts, resolved_slot, ref_fps, latest_vtable_fps, consensus_abi, abi_obs, }; // Checked HERE too, not only in the fold: a distill that emits a desynced model is the same defect // one build later, and the fold's error would then name the wrong producer. check_model_shape(&model)?; let text = serde_json::to_string(&model)?; std::fs::write(out, &text).with_context(|| format!("write {}", out.display()))?; eprintln!( "corpus model: {} builds, {} classes, {} sig fns, {:.1} MB -> {}", model.builds.len(), classes.len(), sig_funcs.len(), text.len() as f64 / 1e6, out.display() ); Ok(()) } /// One build's tracked-class vtable fingerprints/addresses plus each sig fn's uniquely-resolved address — /// the per-build extraction shared by the full distill (`corpus-model`) and the incremental fold. One RTTI /// sweep per image (imgs are [server, engine2, …] — the first image to carry a class wins), keeping only the /// `want` classes' primary vtables: O(sweep + classes), not O(classes × sweep). fn extract_build_vtables( prof: &GameProfile, dir: &Path, want: &HashSet<&str>, sig_funcs: &[&Func], nthreads: usize, ) -> BuildVtables { let (imgs, unreadable) = load_build_images_counted(prof, dir); let (mut fps, mut addrs) = (HashMap::new(), HashMap::new()); let mut truncated: Vec = Vec::new(); for img in &imgs { // Fresh memo per IMAGE (never shared across images — see slot_fingerprints_memo's caveat): inherited // slots repeat across this image's subclass vtables, so this collapses the fingerprint pass. let mut fp_memo: HashMap>> = HashMap::new(); for cv in rtti::enumerate_vtables(img, prof.max_vtable_slots) .into_iter() .filter(|c| c.offset_to_top == 0) { if want.contains(cv.name.as_str()) && !fps.contains_key(&cv.name) { // Exactly at the cap means the read STOPPED there, not that the vtable ended there — the // tail (and any offset in it) is missing from the model with nothing else to show for it. if cv.slots.len() == prof.max_vtable_slots { truncated.push(cv.name.clone()); } fps.insert( cv.name.clone(), slot_fingerprints_memo(img, &cv.slots, &mut fp_memo), ); addrs.insert(cv.name.clone(), cv.slots); } } } if !truncated.is_empty() { truncated.sort(); eprintln!( " WARNING {}: {} class(es) hit the {}-slot vtable cap (truncated — raise \ GameProfile::max_vtable_slots for {}): {}{}", label_of(dir), truncated.len(), prof.max_vtable_slots, prof.token, truncated .iter() .take(5) .cloned() .collect::>() .join(", "), if truncated.len() > 5 { ", …" } else { "" } ); } // `resolve_unique` dominates the whole extract — a `scan_sig_hits` sweep of every sig fn across all the // build's images (the RTTI+fingerprint pass above is a tiny fraction of it). It holds no per-build // fingerprints, so parallelising it over the sig set is RSS-NEUTRAL, and the result is keyed by name so it // is byte-identical to a serial resolve. This is the distill's real hot loop (see the // build-serial driver in `corpus_model_cmd`, which lets this saturate all cores at ~1.5 builds resident). let resolved: HashMap = parallel_map(sig_funcs, nthreads, |f| { resolve_unique(f, &imgs).map(|a| (f.name.clone(), a)) }) .into_iter() .flatten() .collect(); BuildVtables { date: build_date(&label_of(dir)), fps, addrs, resolved, unreadable, empty: imgs.is_empty(), } } /// Unique address of `f` scanning a build's loaded images directly (server+engine2 as a set). fn resolve_unique(f: &Func, imgs: &[CodeImage]) -> Option { // Union the per-image unique-match addresses; resolve iff exactly one distinct address wins across all // images (vote counts are irrelevant here — only the distinct-address set matters). let mut keys: BTreeSet = BTreeSet::new(); for img in imgs { keys.extend(scan_sig_hits(f, img).into_keys()); } (keys.len() == 1).then(|| *keys.iter().next().unwrap()) } /// The vtable slot a sig fn's resolved address lands in for one build — the fourth per-build "fact" the /// model records (beside slot_counts / hops / ref windows). SHARED by the full distill and the incremental /// fold for the same reason its three siblings (`extract_build_vtables`/`ref_obs_of_build`/`tracked_classes`) /// are: the fold claims byte-equality with a re-distill, and an inline copy edited in one path but not the /// other would break that silently. fn resolved_slot_of(bv: &BuildVtables, f: &Func) -> Option { let class = class_of(&f.name); bv.resolved.get(&f.name).and_then(|addr| { bv.addrs .get(class) .and_then(|a| a.iter().position(|x| x == addr)) }) } /// One reference build's per-sig-fn `(name, fingerprint, optional ABI shape)` observations — the ref-window /// extraction shared by the full distill and the incremental fold. Uses `locate_addr` (the same path the /// live-derive verification takes), not the chunk loop's `resolve_unique`. The per-sig work /// (locate_addr + fingerprint + abi_shape, all thread-safe reads over `rimg`) is parallelised `nthreads`-wide: /// in the FOLD this is the sole per-sig pass (a serial 1-wide stretch otherwise), and in the distill the /// caller drives the ref builds SERIALLY so this inner fan-out uses the cores rather than nesting under an /// outer parallel_map. `parallel_map` preserves input order, so the output equals the old serial `filter_map`. fn ref_obs_of_build( prof: &GameProfile, dir: &Path, cat: &[Func], sig_funcs: &[&Func], nthreads: usize, ) -> RefObs { let (rimg, _unreadable) = preload_images(prof, cat, dir); // The identity check belongs HERE, not only where the artifact is emitted. This function is what // teaches the model what a name looks like, and it is shared by the distill and the incremental fold — // so a contradiction rejected only at emit time would leave the model still learning the wrong // function's fingerprint, and the strict fingerprint check would then CONFIRM that wrong resolution on // the next build. Measured: that is exactly how one bad entry survived three guards and shipped. let ident = Identity::of(&rimg); parallel_map(sig_funcs, nthreads, |f| { let (img, addr, _) = locate_addr_ident(prof, f, &rimg, Some(&ident))?; let fp = fingerprint::extract(img, addr)?.to_vec(); Some(( f.name.clone(), fp, abi::abi_shape(img, addr).map(Into::into), )) }) .into_iter() .flatten() .collect() } /// Parse a distilled corpus model from disk. Shared by the standalone `fold-model` command and `produce` /// (which parses it ONCE, lets the derive borrow it, then hands it by value to the sidecar fold). pub fn load_model(path: &Path) -> Result { let text = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; serde_json::from_str(&text).with_context(|| format!("parse {}", path.display())) } /// Fold ONE new build into an existing corpus model: `model N + build N+1 → model N+1`, reading only the /// loaded model + the one new binary (no corpus). The model is DESIGNED for this — `latest_vtable_fps` holds /// the previous build's per-class fingerprints, exactly the seed for the new hop. /// /// EQUIVALENCE — the result equals a full re-distill over the same N+1 builds ONLY when the fold is run with /// the same `class_scope` (asserted below) AND the same catalogue the model was distilled from, AND no class /// re-appears across the model's latest-build boundary (present in old builds, absent from the model's latest, /// present again now). In those cases a class/sig-fn the model has never seen is back-filled with ABSENT /// history (a re-distill would instead resolve its real slots/counts/hops in the old binaries) — reduced /// coverage for that one name until the next full re-distill, never a wrong offset. The catalogue is not /// fingerprint-checked (production always folds with the distill's catalogue), but a newcomer is WARNED below. pub fn fold_model_cmd( prof: &GameProfile, mut model: CorpusModel, catalogue: &Path, build: &Path, class_scope: ClassScope, out: &Path, ) -> Result<()> { let cat = load_catalogue(catalogue)?; let vfuncs = vtable_offset_timelines(&cat); let sig_funcs: Vec<&Func> = cat.iter().filter(|f| !linux_sigs(f).is_empty()).collect(); ensure!( model.class_scope == class_scope, "the input model was distilled with class-scope {:?}, but the fold was called with {:?} — they must \ match or the folded model would diverge from a re-distill; re-run with --class-scope {:?}", model.class_scope, class_scope, model.class_scope ); // The new build becomes the latest, so its RTTI (scope-filtered) ∪ catalogue classes IS the new tracked // set — exactly as a full distill takes `classes` from ITS latest build (hence the SHARED // `tracked_classes`). So a class dropped from the new build's RTTI (and not catalogue-required) falls // out; a class new to it is back-filled below. // SCOPED, for the reason the move-not-clone below exists: `extract_build_vtables` loads this same // build again, so leaving these alive would peak at TWO complete image sets and then hold the dead // one through the model rebuild and its multi-hundred-MB serialization. let classes = { let latest_imgs = load_build_images(prof, build); tracked_classes(prof, class_scope, &latest_imgs, &vfuncs, &sig_funcs) }; let want: HashSet<&str> = classes.iter().map(String::as_str).collect(); let bv = extract_build_vtables(prof, build, &want, &sig_funcs, default_threads(None)); // Same guard the full distill enforces: a build that loaded zero libraries contributes empty // fingerprints → empty hops → an unconditional chain break severing every anchor older than it. The // fold is the standing production path (the sidecar in every `produce`) and its output SEEDS the next // build's derive, so folding damage in silently is worse here than in the distill, not better. ensure!( !bv.empty, "build {} loaded ZERO libraries — folding it would sever every anchor chain crossing it; \ repair or skip this build", bv.date ); if !bv.unreadable.is_empty() { eprintln!( " WARNING build {}: {} unparseable ({}) — its fingerprints are partial", bv.date, bv.unreadable.len(), bv.unreadable.join(", ") ); } let old_nb = model.builds.len(); // Per class: append the new build's slot count and ONE hop. A continuing class extends its history; a // class new to this build is back-filled with absent history (0 count / empty hop for every prior build), // matching what a re-distill produces for a class first seen at build N. let mut hops: BTreeMap> = BTreeMap::new(); let mut slot_counts: BTreeMap> = BTreeMap::new(); let mut backfilled = 0usize; // classes/sig-fns the model has never seen (see the EQUIVALENCE note above) for c in &classes { let new_count = bv.fps.get(c).map_or(0, Vec::len); // MOVE the class's prior counts/hops out of the owned model instead of cloning them (the model is // dropped at end of fn and `hops` is its size-dominant field): `remove` returns the exact Vec a // clone would have produced, and nothing reads `model.slot_counts`/`model.hops` after this loop, so // the folded model is byte-identical. Halves the fold's transient peak RSS. match model.slot_counts.remove(c) { Some(mut counts) => { counts.push(new_count); slot_counts.insert(c.clone(), counts); // the hop into the new build = align the model's stored latest fps against the new build's // (empty where either side lacks the class — identical to the distill's windows(2) pairing). let hop: Vec = match (model.latest_vtable_fps.get(c), bv.fps.get(c)) { (Some(a), Some(b)) => nw_align(a, b).into_iter().map(Slot::from_opt).collect(), _ => Vec::new(), }; let mut h = model.hops.remove(c).unwrap_or_default(); h.push(Hop::from_slots(hop)); hops.insert(c.clone(), h); } None => { backfilled += 1; let mut counts = vec![0usize; old_nb]; counts.push(new_count); slot_counts.insert(c.clone(), counts); hops.insert(c.clone(), vec![Hop::default(); old_nb]); // old_nb == new_nb − 1 hops, all empty } } } // Per sig fn: append the vtable slot its resolved address lands in this build (None if unresolved). // `resolved_slot` is a per-build parallel array read by build INDEX (`build_per_build_vtables`), so a // name absent from the model — a class/sig-fn the CATALOGUE gained since the distill (NOT a contribution: // contributions are a derive-time overlay and never enter the model) — must be back-filled with absent // history first, exactly as the class loop above. Without it the lone new observation lands at index 0 // and is attributed to the OLDEST build in the corpus. (Back-fill ≠ a re-distill's real old-build history // for such a newcomer — see the EQUIVALENCE note on `fold_model_cmd`; counted + warned below.) let mut resolved_slot: BTreeMap> = BTreeMap::new(); for f in &sig_funcs { // move-out (see the class loop): the removed Vec == the cloned one; nothing reads the field after. let mut v = model.resolved_slot.remove(&f.name).unwrap_or_else(|| { backfilled += 1; vec![Slot::NONE; old_nb] }); v.push(Slot::from_opt(resolved_slot_of(&bv, f))); resolved_slot.insert(f.name.clone(), v); } // Ref/ABI windows: prepend the new build's observations (newest-first), then evict everything outside // the newest CORPUS_REF_K BUILDS. Evicting by build age — not by list length — is what makes this equal // a re-distill: a fn that fails to resolve in some builds would otherwise retain fingerprints from // arbitrarily far back, and the divergence would compound over every future fold. let new_idx = old_nb as u32; // index of the build being appended = the new latest let cutoff = new_idx.saturating_sub(CORPUS_REF_K as u32 - 1); // keep indices >= cutoff let mut ref_fps = std::mem::take(&mut model.ref_fps); let mut abi_obs = std::mem::take(&mut model.abi_obs); for (n, fp, sh) in ref_obs_of_build(prof, build, &cat, &sig_funcs, default_threads(None)) { ref_fps .entry(n.clone()) .or_default() .insert(0, (new_idx, fp)); if let Some(s) = sh { abi_obs.entry(n).or_default().insert(0, (new_idx, s)); } } for v in ref_fps.values_mut() { v.retain(|(i, _)| *i >= cutoff); } for v in abi_obs.values_mut() { v.retain(|(i, _)| *i >= cutoff); } ref_fps.retain(|_, v| !v.is_empty()); abi_obs.retain(|_, v| !v.is_empty()); let consensus_abi = consensus_from_windows(&abi_obs); let latest_vtable_fps: BTreeMap>>> = bv.fps.into_iter().collect(); let mut builds = model.builds; builds.push(BuildMeta { label: label_of(build), date: build_date(&label_of(build)), }); let folded = CorpusModel { class_scope, builds, hops, slot_counts, resolved_slot, ref_fps, latest_vtable_fps, consensus_abi, abi_obs, }; check_model_shape(&folded)?; let out_text = serde_json::to_string(&folded)?; std::fs::write(out, &out_text).with_context(|| format!("write {}", out.display()))?; eprintln!( "folded {} -> model N+1: {} builds, {} classes, {} sig fns, {:.1} MB -> {}", label_of(build), folded.builds.len(), classes.len(), sig_funcs.len(), out_text.len() as f64 / 1e6, out.display() ); if backfilled > 0 { eprintln!( " NOTE {backfilled} class(es)/sig-fn(s) were new to the model and back-filled with EMPTY \ pre-fold history — a re-distill would record their real old-build history instead (catalogue \ grew since the distill). Harmless (reduced coverage for those names only); re-distill for exact." ); } Ok(()) } // ══════════════════════════════════════════════════════════════════════════════════════════════ // §7 · GAMEDATA DERIVATION (catalogue → signatures / vtable offsets) // ══════════════════════════════════════════════════════════════════════════════════════════════ /// A per-catalogue-function signature-resolution verdict from [`resolve_signatures`]. enum SigResolve { Skip, Flag(String), Emit(String, String, String), // confident: (name, library, signature) EmitFallback(String, String, String), // heuristic-flagged but uniquely resolves — live-validate /// The address resolved, and the BINARY CONTRADICTS the name there — `(name, why)`. Kept distinct from /// `Flag` because it is a different fact and a louder one: a drifted sig failed to find its function, /// while this one found a function that is provably not it. Contradicted(String, String), } /// What a candidate address is checked AGAINST, in one place because the three travel together: the /// cross-build fingerprint history and the bar it has to clear, plus what the binary itself states about /// the address ([`Identity`]). Fingerprints answer "is this the same function as last build"; `Identity` /// answers "is this the function this name means at all" — different questions, both needed. struct Verify<'a> { ref_fps: HashMap>>, ref_majority: usize, ident: &'a Identity, } /// `(dbg line, resolution, optional (name, abi-drift detail), optional (name, derived ABI shape))` per /// catalogue function. type SigItem = ( Option, SigResolve, Option<(String, String)>, Option<(String, model::AbiShape)>, ); /// The engine's decoded shape as the shipped, format-neutral one. Kept a plain fn rather than a `From` impl /// because both types are named `AbiShape` and the conversion crosses the crate boundary in one direction. fn shipped_abi(s: abi::AbiShape) -> model::AbiShape { model::AbiShape { int: s.int_args, float: s.float_args, stack: s.stack_args, ret: s.ret_class.describe().to_string(), } } /// Resolve every catalogue signature in the target, across threads: locate the era-sig's candidates, verify /// the hit against the reference fingerprints (or a self-consensus when history is thin), regenerate a fresh /// unique sig, and run the derive-time ABI-drift check. Read-only over `images`/`ref_fps`; the caller applies /// the returned verdicts in catalogue order, so `gd`/`sig_ok`/`sig_flag` stay byte-identical to a serial run. fn resolve_signatures( prof: &GameProfile, cat: &[Func], images: &HashMap, v: &Verify<'_>, cmodel: Option<&CorpusModel>, dbg: bool, ) -> Vec { let (ref_fps, ref_majority, ident) = (&v.ref_fps, v.ref_majority, v.ident); const VERIFY_L1_MAX: u64 = 12; parallel_map(cat, default_threads(None), |f| { if linux_sigs(f).is_empty() { return (None, SigResolve::Skip, None, None); } let Some((img, lib, cands)) = locate_candidates(prof, f, images) else { return (None, SigResolve::Flag(f.name.clone()), None, None); }; let mut dbg_line = None; let chosen = match ref_fps.get(&f.name) { // stable recent history: pick the target candidate nearest the real function's fingerprint, // accept only within recompile distance (rejects a stale sig that drifted onto a decoy). Some(rs) if rs.len() >= ref_majority => { let best = cands .iter() .filter_map(|&(addr, _)| { let tfp = fingerprint::extract(img, addr)?.to_vec(); let d = rs.iter().map(|r| l1(&tfp, r)).min().unwrap_or(u64::MAX); Some((addr, d)) }) .min_by_key(|&(_, d)| d); if dbg && let Some((a, d)) = best { dbg_line = Some(format!( " [dbg] {:<46} @{a:#x} L1={d} ({} refs)", f.name, rs.len() )); } best.filter(|&(_, d)| d <= VERIFY_L1_MAX).map(|(a, _)| a) } // no stable reference history: trust only a self-corroborating consensus of >=2 distinct era-sigs // agreeing in the target itself; a lone hit is otherwise a likely coincidental decoy -> flag. _ => { if dbg { let top = cands.first().map(|&(a, v)| (a, v)).unwrap_or((0, 0)); dbg_line = Some(format!( " [dbg] {:<46} @{:#x} votes={} refs={} -> {}", f.name, top.0, top.1, ref_fps.get(&f.name).map_or(0, Vec::len), if top.1 >= 2 { "consensus" } else { "FLAG" } )); } cands.first().filter(|&&(_, v)| v >= 2).map(|&(a, _)| a) } }; // Fallback: the verification heuristic rejected every candidate, but if exactly one era-sig hit // uniquely resolves in the target it is a legitimate — if unverified — location; emit it and lean on // validate-live as the real gate rather than dropping a real function that merely drifted. let was_flagged = chosen.is_none(); let chosen = chosen.or_else(|| (cands.len() == 1).then(|| cands[0].0)); // Refuse an address the binary itself says is a different function — including one the fallback // above would otherwise emit, since "exactly one era-sig resolves" is precisely how a stale sig's // coincidental hit gets through. if let Some(addr) = chosen && let Some(why) = ident.contradiction(&lib, img, addr, &f.name) { return ( None, SigResolve::Contradicted(f.name.clone(), why), None, None, ); } // Derive-time prototype-drift check (model path only): unknown-return-guarded, a kept signal for // review, not a drop — the sig still emits. // Measured for EVERY resolved signature, not just those with corpus history: it is what lets a // declared prototype be machine-checked against this build, and the drift check below is a second // reading of the same measurement rather than a separate one. let shape = chosen.and_then(|addr| abi::abi_shape(img, addr)); let abi_drift = shape.and_then(|s| { let cons = cmodel?.consensus_abi.get(&f.name)?; let sh: AbiSig = s.into(); sh.differs(cons).then(|| { ( f.name.clone(), format!("target [{}] vs history [{}]", sh.brief(), cons.brief()), ) }) }); // The measured footprint ships ONLY where the signature does, and the pairing is what makes it // meaningful: a `Flag` means this address produced no signature, so the name goes on to // `sig_flag` where `recover_by_string_anchor` / `recover_virtual_sigs` may relocate it. `make_sig` // is deterministic per (image, address) and already failed here, so any later recovery is // provably at a DIFFERENT address — and a footprint measured at this one would then describe some // other function while `prototypes::agrees` judged declared types against it. No shape is the // acceptable degradation; a shape belonging elsewhere is the thing this project forbids. let (r, shape_out) = match chosen .and_then(|addr| emit::make_sig(img, addr, CORE_SIG_CAP).map(|s| (lib, s))) { Some((lib, sig)) if was_flagged => ( SigResolve::EmitFallback(f.name.clone(), lib, sig), shape.map(|s| (f.name.clone(), shipped_abi(s))), ), Some((lib, sig)) => ( SigResolve::Emit(f.name.clone(), lib, sig), shape.map(|s| (f.name.clone(), shipped_abi(s))), ), None => (SigResolve::Flag(f.name.clone()), None), }; (dbg_line, r, abi_drift, shape_out) }) } /// Byte budget for signatures generated by the DERIVE (the `core` tier). Distinct from `--sig-cap`, which /// bounds the FOLD's generated sigs: the two run over different inputs, and the derive's value is the one /// the shipped guaranteed tier was calibrated against. const CORE_SIG_CAP: usize = 400; /// How many of the newest reference builds the sig-hit verification samples when it reads BINARIES; also the /// history FLOOR (`/2 + 1` = 3) below which the model path won't trust a fingerprint match. See /// [`build_ref_fingerprints`] for why the model path reuses this constant rather than `CORPUS_REF_K`. const REF_BUILDS: usize = 5; /// Reference fingerprints for the sig-hit verification, plus the clean-resolve majority threshold: straight /// from the corpus MODEL (which ships them), or by resolving + fingerprinting each sig fn in the reference /// binaries. (A function resolving cleanly in the majority of sampled builds == a stable recent history; /// below that a lone target hit is a suspected decoy.) fn build_ref_fingerprints( prof: &GameProfile, cmodel: Option<&CorpusModel>, cat: &[Func], ref_dirs: &[PathBuf], ) -> (HashMap>>, usize) { if let Some(m) = cmodel { // the model ships ref_fps as a (deterministic) BTreeMap; the derive-side verification set is a plain // HashMap (order-independent — it's only `.get()`/`.len()`), so copy it across. let ref_fps = m .ref_fps .iter() .map(|(k, v)| (k.clone(), v.iter().map(|(_, fp)| fp.clone()).collect())) .collect(); // This threshold is a HISTORY FLOOR, not a per-window majority — deliberately `REF_BUILDS/2+1` (=3), // NOT `CORPUS_REF_K/2+1` (=5), even though the model's window is CORPUS_REF_K long. `resolve_signatures` // routes on `window.len() >= threshold`: at/above it a function takes the STRICT arm, which accepts the // target only within recompile distance of its NEAREST historical fingerprint (`rs.iter().map(l1).min()` // — one close ref suffices; it is not a vote). So "resolved cleanly in >=3 of the newest builds" is // ample basis for that min-distance check. A higher floor would demote functions with only 3-4 builds // of genuine history to the lenient >=2-era-sig arm, where single-candidate ones fall through to the // UNVERIFIED fallback — so a higher floor yields MORE unverified entries, not fewer. return (ref_fps, REF_BUILDS / 2 + 1); } let per_ref: Vec)>> = parallel_map(ref_dirs, default_threads(None), |dir| { let (rimg, _unreadable) = preload_images(prof, cat, dir); let mut v = Vec::new(); for f in cat { if linux_sigs(f).is_empty() { continue; } if let Some((img, addr, _)) = locate_addr(prof, f, &rimg) && let Some(fp) = fingerprint::extract(img, addr) { v.push((f.name.clone(), fp.to_vec())); } } v }); let mut rf: HashMap>> = HashMap::new(); for v in per_ref { for (n, fp) in v { rf.entry(n).or_default().push(fp); } } let maj = ref_dirs.len() / 2 + 1; (rf, maj) } /// Apply the (thread-resolved) per-function sig verdicts to `gd` IN CATALOGUE ORDER, so `gd` (name-keyed), /// the emitted-count and the flag/fallback lists are byte-identical to a serial resolve. Emits each captured /// DBG line and reports the fallback set. Returns `(emitted, flagged, fallback, abi_drift)` — `flagged`/ /// `emitted` are further mutated by the recovery passes in the caller. fn apply_sig_items( sig_items: Vec, gd: &mut model::Gamedata, ) -> ( usize, Vec, Vec, Vec, BTreeMap, ) { let mut sig_ok = 0; let mut sig_flag: Vec = Vec::new(); let mut sig_fallback: Vec = Vec::new(); let mut contradicted: Vec = Vec::new(); let mut abi_drift: Vec = Vec::new(); let mut shapes: BTreeMap = BTreeMap::new(); for (dbg_line, r, drift, shape) in sig_items { if let Some((name, sh)) = shape { shapes.insert(name, sh); } if let Some(l) = dbg_line { eprintln!("{l}"); } if let Some((name, detail)) = drift { abi_drift.push(model::Flagged::new( name, model::FlagReason::AbiDrift, detail, )); } match r { SigResolve::Skip => {} SigResolve::Flag(name) => sig_flag.push(name), SigResolve::Contradicted(name, why) => { // Carried with its OWN reason rather than merged into the drifted name list: the artifact // must not tell a consumer "no signature resolved" when one did and was refused. It rides // the `Flagged` channel (which already carries a reason per entry) rather than widening // this function's return. contradicted.push(model::Flagged::new( name, model::FlagReason::NameContradicted, why, )); } SigResolve::Emit(name, lib, sig) => { gd.set_signature(name, lib, sig); sig_ok += 1; } SigResolve::EmitFallback(name, lib, sig) => { gd.set_signature(name.clone(), lib, sig); sig_ok += 1; sig_fallback.push(name); } } } if !contradicted.is_empty() { eprintln!( " {} signatures REFUSED — the binary contradicts the name at the address they resolve to \ (they ship as `unresolved`, not as a locator):", contradicted.len() ); for c in &contradicted { eprintln!(" {} — {}", c.name, c.detail); } } abi_drift.extend(contradicted); if !sig_fallback.is_empty() { eprintln!( " {} signatures emitted via unique-resolve fallback (fingerprint-UNVERIFIED — marked \ `catalogue-unverified` in the monolith): {}", sig_fallback.len(), sig_fallback.join(", ") ); } (sig_ok, sig_flag, sig_fallback, abi_drift, shapes) } /// One offline derive, in memory — no files are written; the fold consumes this directly, disk-free. pub(crate) struct Derived { /// The cssharp-rendered guaranteed gamedata (the `core` tier). pub core: String, /// Why each catalogue entry failed to produce a shipped locator. Typed, not a JSON string: this crosses /// no disk boundary (the fold consumes it in the same call chain), and a typed value makes a renamed /// field a compile error rather than a silently absorbed default. pub flagged: Vec, /// Names emitted through the unique-resolve fallback: the reference-fingerprint check REJECTED every /// candidate, but exactly one era-sig resolved uniquely, so the locator is sound while the identity is /// UNVERIFIED. They land in `core` like any other catalogue entry, so the monolith must mark them — /// live validation cannot re-verify them (it checks that a sig resolves to executable code, and the /// pattern was generated from that very address, so a decoy passes trivially). pub unverified: BTreeSet, /// Per-entry argument footprint measured in the target binary — the machine half of the /// locator/prototype split, and what a declared prototype is checked against. pub abi: BTreeMap, /// Per-entry string ANCHORS from the catalogue — distinctive literals the function references. /// /// Its own side table for the same reason `abi` is: the derive→fold transport for `core` is the cssharp /// locator shape, which has no anchor field, so anything ridden in on an `Entry` there would be dropped /// when the fold re-parses it. Carrying them separately keeps the cssharp artifact unchanged — CS# has /// no `refs` feature and should not grow a key it cannot read. pub anchors: BTreeMap>, } /// Derive a target build's gamedata OFFLINE and return it in memory. pub(crate) fn gamedata( prof: &GameProfile, catalogue: &Path, source: CorpusSource, target: &Path, ) -> Result { let emitter = render::by_id("cssharp").expect("cssharp is a built-in format"); let mut cat = load_catalogue(catalogue)?; // Merge contributions BY NAME into the catalogue, not `extend` — a contribution for a name the catalogue // already carries must add its dated variant to that Func's `variants` (so `scan_sig_hits`, which votes // across one Func's variants, can let it corroborate the existing era-sigs), not append a second Func of // the same name that the resolver would treat as an unrelated entry and double-count in the tallies. for c in load_contributions(prof, catalogue) { match cat.iter_mut().find(|f| f.name == c.name) { Some(f) => f.variants.extend(c.variants), None => cat.push(c), } } let mut gd = model::Gamedata { game_key: prof.game_key.to_string(), ..Default::default() }; // dated vtable-offset timelines (consumed by the offset section below). let vfuncs = vtable_offset_timelines(&cat); // Corpus MODEL (distilled) OR the raw corpus binaries: with a model, only the target binary is // read; the model supplies the reference fingerprints + the cross-build vtable alignment. The model // is pre-parsed by the caller (borrowed here, read-only) so `produce` parses it a single time. let cmodel: Option<&CorpusModel> = match source { CorpusSource::Model(m) => Some(m), CorpusSource::Binaries(_) => None, }; // The corpus model is a FORWARD-derivation artifact: it ships cross-build vtable hops plus the // fingerprints of its LATEST build only, so the closing alignment hop (model's latest -> target) // is reconstructable exactly when the target is newer than every build the model distilled. A // target inside the model's range would both lack the fingerprints for its own predecessor hop // and let post-target builds pollute the offset chains (the slot would drift the wrong way) — so // refuse it up front and point at --corpus rather than emit a quietly-wrong offset. Compare full // "YYYY-MM-DD_HHMMSS" labels (they sort chronologically) so a same-day-but-later build still counts. if let Some(m) = cmodel && let Some(latest) = m.builds.last() { let tlabel = label_of(target); ensure!( tlabel.as_str() > latest.label.as_str(), "target build {tlabel} is not newer than the corpus model's latest build {} — \ --corpus-model derives forward-only. Re-distill a model excluding builds >= the \ target, or derive this target from --corpus .", latest.label ); } // reference builds strictly before the target's date (binary path only). Empty with a model — // which also auto-skips context recovery below (it has no reference binary to build an xref over). let ref_builds: Vec<(String, PathBuf)> = match source { CorpusSource::Model(_) => Vec::new(), CorpusSource::Binaries(corpus) => { let tdate = build_date(&label_of(target)); let mut rb: Vec<(String, PathBuf)> = find_builds(prof, corpus)? .into_iter() .map(|p| (build_date(&label_of(&p)), p)) .filter(|(d, _)| d.as_str() < tdate.as_str()) .collect(); rb.sort(); rb } }; // ---- signatures: resolve a known sig in the target, VERIFY the hit is really the function // (fingerprint vs the newest reference builds — a stale era-sig can uniquely match the // WRONG function), then regenerate a fresh unique sig. ---- let (images, unreadable_libs) = preload_images(prof, &cat, target); // Named loudly: every catalogue entry pointing at one of these will resolve nowhere and ship as // `SigDrifted`, whose detail says a signature drifted in the target — a false CAUSE for a file that // was never opened. The reason has to reach the log even though it cannot reach every flag detail. if !unreadable_libs.is_empty() { eprintln!( " WARNING {} target librar{} PRESENT but unreadable — every catalogue entry naming one \ scans nothing and is flagged as drifted, which is not why it failed: {}", unreadable_libs.len(), if unreadable_libs.len() == 1 { "y is" } else { "ies are" }, unreadable_libs.join(", ") ); } // The target must be a build DIRECTORY that `find_file` can search (a bare `.so` path is NOT searched — // read_dir on a file yields nothing). Without this, a mistyped/file target loads zero images and the // derive silently emits an all-flagged, near-empty release that even clears the live gate. Fail loudly // instead — unless the catalogue is offset-only (no sigs to locate), where empty images is legitimate. ensure!( images.contains_key(prof.server_lib) || cat.iter().all(|f| linux_sigs(f).is_empty()), "target {} loaded no {} — {}", label_of(target), prof.server_lib, // Two very different causes, and telling the operator the wrong one costs an hour: a mistyped or // file-shaped target searches nothing, while a present-but-corrupt server lib is a broken input. if unreadable_libs .iter() .any(|u| u.starts_with(prof.server_lib)) { "it is PRESENT but failed to parse (see the warning above) — truncated, or not an ELF" } else { "pass the build DIRECTORY (a bare .so path is not searched)" } ); // Sample the newest reference builds (there a sig still lands on the real function) to reject a target // hit that is structurally a different function; see build_ref_fingerprints. let ref_dirs: Vec = ref_builds .iter() .rev() .take(REF_BUILDS) .map(|(_, p)| p.clone()) .collect(); let (ref_fps, ref_majority) = build_ref_fingerprints(prof, cmodel, &cat, &ref_dirs); let dbg = std::env::var("SOURCE2ROSETTA_DBG").is_ok(); // Each catalogue function resolves independently — locate its candidates, verify the hit against the // reference fingerprints, and emit a fresh sig — all read-only over images/ref_fps, each doing many // whole-binary memchr scans. Resolve across threads (capturing any SOURCE2ROSETTA_DBG line), then apply in // catalogue order so gd (name-keyed), the counts and the flag list are byte-identical to a serial run. // Built once over the whole image set and shared read-only across the resolution threads: both halves // are whole-image passes, so recomputing per function would dominate the derive. let ident = Identity::of(&images); let verify = Verify { ref_fps, ref_majority, ident: &ident, }; let sig_items = resolve_signatures(prof, &cat, &images, &verify, cmodel, dbg); let (mut sig_ok, mut sig_flag, unverified, abi_drift, mut abi_shapes) = apply_sig_items(sig_items, &mut gd); // ---- one full-corpus vtable pass feeds BOTH offset chaining and vtable-anchoring recovery. // Both need each class's vtable fingerprinted across every build; doing it once (over the // union of the classes they need) halves the work versus two separate passes. ---- // still-flagged functions whose class has a target vtable = virtual-anchoring candidates let virt_funcs: Vec<(&Func, String)> = cat .iter() .filter(|f| !linux_sigs(f).is_empty() && sig_flag.iter().any(|n| n == &f.name)) .map(|f| (f, class_of(&f.name).to_string())) .filter(|(_, c)| { images .values() .any(|im| rtti::find_vtable(im, c, prof.max_vtable_slots).is_some()) }) .collect(); let classes: Vec = vfuncs .iter() .map(|f| f.class.clone()) .chain(virt_funcs.iter().map(|(_, c)| c.clone())) .collect::>() .into_iter() .collect(); // per-build vtable data (fingerprints + addresses + resolved anchors): from the MODEL (only the // target binary is read; past builds are synthesised from the distilled signals) or by // fingerprinting every reference build's binaries. let (per_build, target_idx) = build_per_build_vtables(prof, cmodel, ref_builds, target, &classes, &virt_funcs); // vtable-slot alignment across consecutive builds: the model ships past hops; the final hop // (latest model build -> target) is computed from the target's real vtable fingerprints. let hops = build_hops(cmodel, &per_build, target_idx, &classes); // vtable-anchoring recovery for changed virtual functions (shares the pass above) let vrec = recover_virtual_sigs( prof, &virt_funcs, &per_build, &hops, target_idx, &images, &mut gd, ); if !vrec.is_empty() { sig_ok += vrec.len(); // Re-measured AT THE ADDRESS RECOVERY LANDED ON. The resolver ships no footprint for a flagged // name precisely because recovery moves it, so this is what puts one back — measured where the // signature now points rather than where it used to. for (name, lib, addr) in &vrec { if let Some(sh) = images.get(*lib).and_then(|im| abi::abi_shape(im, *addr)) { abi_shapes.insert(name.clone(), shipped_abi(sh)); } } let names: Vec<&str> = vrec.iter().map(|(n, ..)| n.as_str()).collect(); sig_flag.retain(|n| !names.contains(&n.as_str())); eprintln!( " {} drifted signatures RECOVERED via vtable anchoring: {}", vrec.len(), names.join(", ") ); } // ---- string-anchor recovery: functions carrying a distinctive search-string (a sig-list anchor) // are relocated in the target by that string and a fresh sig emitted — re-resolved per build, // so the anchor survives byte-sig drift. Recovers anchors with no byte sig AND still-flagged // ones that also carry an anchor. ---- let arec = recover_by_string_anchor(prof, &cat, &images, &mut gd); if !arec.is_empty() { sig_ok += arec.len(); // Same re-measurement, same reason. String anchors are server-lib by construction. if let Some(im) = images.get(prof.server_lib) { for (name, addr) in &arec { if let Some(sh) = abi::abi_shape(im, *addr) { abi_shapes.insert(name.clone(), shipped_abi(sh)); } } } let names: Vec<&str> = arec.iter().map(|(n, _)| n.as_str()).collect(); sig_flag.retain(|n| !names.contains(&n.as_str())); eprintln!( " {} functions RECOVERED by string anchor: {}", arec.len(), names.join(", ") ); } // ---- offsets: chain from dated corpus reference builds to the target, then vote ---- let (off_ok, off_flag) = derive_offsets(&vfuncs, &hops, &per_build, target_idx, &mut gd); let core = emitter.render(&gd); eprintln!( "gamedata for {}: {sig_ok} signatures + {off_ok} offsets emitted", label_of(target) ); if !sig_flag.is_empty() { eprintln!( " {} signatures FLAGGED (drifted): {}", sig_flag.len(), sig_flag.join(", ") ); } if !off_flag.is_empty() { let rendered: Vec = off_flag .iter() .map(|f| format!("{} ({})", f.name, f.detail)) .collect(); eprintln!( " {} offsets FLAGGED (review): {}", off_flag.len(), rendered.join(", ") ); } // Scoped to the drift reason: this vec also carries the name-contradiction refusals, which are a // different fact and are reported where they are found. let drifted: Vec<&model::Flagged> = abi_drift .iter() .filter(|f| f.reason == model::FlagReason::AbiDrift) .collect(); if !drifted.is_empty() { // A prototype-drift WARNING (sig still emitted): the target's ABI shape differs from the model's // consensus — the loader-hook seam the byte-sig can't see, caught forward without the corpus. eprintln!( " {} signatures with ABI DRIFT vs model consensus (kept; review prototype): {}", drifted.len(), drifted .iter() .map(|f| format!("{} ({})", f.name, f.detail)) .collect::>() .join(", ") ); } // The flag list: sig-drifted + offset-review + abi-drift, for the monolith's unresolved tier. let mut flagged: Vec = sig_flag .into_iter() .map(|n| { model::Flagged::new( n, model::FlagReason::SigDrifted, "no unique/recovered signature in target", ) }) .collect(); flagged.extend(off_flag); flagged.extend(abi_drift); // ---- the completeness sweep, and it is the thing that makes `unresolved` a CONTRACT rather than a // by-product. Every pass above flags what IT could not place: the sig resolver flags a drifted // signature, the offset pass flags a low-confidence vote. But a name reaches those passes only if // it has the input they consume, and one class of entry has neither — an ANCHOR-ONLY catalogue // entry (476 of CS2's 1,601) never enters the sig resolver at all, so when its anchor finds zero // or several xref hits it lands in no tier and no flag list: it simply stops existing, which is // the one outcome README.md ("it never just disappears") and CONTRIBUTING.md both rule out. // Sweeping the catalogue itself, rather than adding a fourth per-pass flag, is what makes the // claim hold for the NEXT locator kind too. ---- let accounted: HashSet<&str> = gd .entries .keys() .map(String::as_str) .chain(flagged.iter().map(|f| f.name.as_str())) .collect(); let by_name: HashMap<&str, &Func> = cat.iter().map(|f| (f.name.as_str(), f)).collect(); let mut vanished: Vec<&str> = by_name .keys() .copied() .filter(|n| !accounted.contains(n)) .collect(); vanished.sort_unstable(); if !vanished.is_empty() { eprintln!( " {} catalogue entries produced no locator and no flag — recorded as unresolved: {}", vanished.len(), vanished.join(", ") ); flagged.extend(vanished.into_iter().map(|n| { model::Flagged::new( n, model::FlagReason::Unresolved, unresolvable_because(by_name[n]), ) })); } // Every catalogue entry's anchors, whether or not a byte sig located it. `recover_by_string_anchor` // uses them only as a FALLBACK locator; this ships them as a supplement, which is a different job — an // entry that resolved perfectly still benefits from a second locator with a different failure mode. let anchors: BTreeMap> = cat .iter() .filter_map(|f| { let a: Vec = string_anchors(f).into_iter().map(str::to_string).collect(); (!a.is_empty()).then(|| (f.name.clone(), a)) }) .collect(); Ok(Derived { core, flagged, unverified: unverified.into_iter().collect(), abi: abi_shapes, anchors, }) } /// One build's vtable data, shared by offset chaining and vtable-anchoring recovery. #[derive(Default)] struct BuildVtables { date: String, fps: VtableFps, // class -> slot fingerprints (for chaining) addrs: HashMap>, // class -> slot addresses (to read the located slot) resolved: HashMap, // anchoring candidate name -> resolved address /// Libraries present in the build but unparseable. Empty on a healthy build; non-empty means this /// build's fingerprints are partial, which silently breaks anchor chains through it. unreadable: Vec, /// True when no library loaded at all — the build contributes nothing but still occupies an index. empty: bool, } /// Recover functions by a distinctive string they reference (a `string-anchor` locator from a curated sig /// list). Unlike byte sigs, the anchor string is stable across builds, so this re-resolves /// per build: for each catalogue function with a string-anchor not already located by a byte sig, take /// the *unique* function referencing that string in the target and emit a fresh sig there. A string /// referenced by ≥2 functions is not distinctive enough → skipped (stays flagged). Server-only (all /// curated anchors are server-lib). Returns each recovered name WITH the address it was relocated to — /// the caller re-measures the footprint there, because the one measured before recovery belonged to an /// address this build rejected. fn recover_by_string_anchor( prof: &GameProfile, cat: &[Func], target_images: &HashMap, gd: &mut model::Gamedata, ) -> Vec<(String, u64)> { let anchored: Vec<&Func> = cat .iter() .filter(|f| !string_anchors(f).is_empty()) .collect(); let Some(img) = target_images.get(prof.server_lib) else { return Vec::new(); }; if anchored.is_empty() { return Vec::new(); } let idx = xref::XrefIndex::build(img); let mut recovered = Vec::new(); let mut unreachable = 0usize; for f in anchored { if gd.entries.contains_key(&f.name) { continue; // already located by a byte sig — the anchor is just a backup } for s in string_anchors(f) { let hits = xref::funcs_using_string(img, &idx, s); let [addr] = hits.as_slice() else { continue }; // "Exactly one function references it" is attribution by `containing_func` — nearest entry // at-or-below — and these binaries do not support that on its own: CS2 strips `.eh_frame` // from game code, so a `[entry, next_entry)` range routinely spans a real function plus // unindexed neighbours and inherits their strings. `attach_derived_anchors` spends two // conditions on exactly this hazard when it DERIVES an anchor; using one to locate a // function is the same claim in the other direction and gets the same test: every // instruction that loads the string must sit inside the candidate's own flow-reachable // code. Where it does not, the real referrer is unindexed and this "unique" hit is its // preceding neighbour — a signature that resolves, validates live, and names the wrong // function, shipped into the guaranteed tier. Dropping it is the honest degradation. let Some((reach, _)) = reachable_strings(img, *addr) else { continue; }; let attributable = img .find_bytes(s.as_bytes()) .into_iter() .flat_map(|va| idx.refs_to(va)) .all(|ip| reach.contains(ip)); if !attributable { unreachable += 1; continue; } if let Some(sig) = emit::make_sig(img, *addr, CORE_SIG_CAP) { gd.set_signature(f.name.clone(), "server", sig); recovered.push((f.name.clone(), *addr)); break; } } } if unreachable > 0 { eprintln!( " string-anchor recovery: {unreachable} anchors REJECTED — the string is loaded from \ outside the candidate's reachable code, so the unique-referrer hit is a neighbouring \ function rather than the one the anchor names" ); } recovered } /// Chain each anchor's slot forward through the vtable-alignment hops `hv` to the target build, then take /// a RECENCY-WEIGHTED vote on the resulting target slot: each anchor's vote is weighted by its build index, /// so a recent anchor — which chains through fewer hops and reflects the current build's slot convention — /// outweighs a distant one whose value may predate a layout shift or come from a differently-indexed tool. /// An offset consistent across history is unaffected, since every anchor chains to the same slot. /// Returns the winning (slot, confidence%), or None if no anchor chained through. fn chain_and_vote( anchors: &[(usize, usize)], hv: &[Vec>], target_idx: usize, ) -> Option<(usize, u64)> { let mut votes: HashMap = HashMap::new(); for &(a, off0) in anchors { let mut pos = Some(off0); for t in a..target_idx { pos = match hv.get(t) { Some(m) if !m.is_empty() => pos.and_then(|p| m.get(p).copied().flatten()), _ => None, }; if pos.is_none() { break; } } if let Some(p) = pos { *votes.entry(p).or_default() += a as u64 + 1; } } let total: u64 = votes.values().sum(); let (&slot, &agree) = votes .iter() .max_by_key(|e| (*e.1, std::cmp::Reverse(*e.0)))?; Some((slot, 100 * agree / total.max(1))) } /// Time-spread vtable anchors: walk reference builds `[0, target_idx)`, keeping at most one anchor per /// 15-build window, where `slot_of(build)` yields the function's slot in that build. The two vtable paths /// (offset chaining + drifted-sig recovery) differ only in HOW they read a build's slot, so this captures the /// shared spacing loop; the closure supplies the per-build read. fn spread_anchors( per_build: &[BuildVtables], target_idx: usize, mut slot_of: impl FnMut(&BuildVtables) -> Option, ) -> Vec<(usize, usize)> { let mut anchors: Vec<(usize, usize)> = Vec::new(); for (i, bv) in per_build.iter().enumerate().take(target_idx) { if anchors.last().is_some_and(|&(li, _)| i - li < 15) { continue; } if let Some(off) = slot_of(bv) { anchors.push((i, off)); } } anchors } /// Derive each catalogued vtable offset for the target build: chain its dated reference offsets forward /// through the per-build slot-alignment hops, then take a recency-weighted vote. Only a high-confidence /// chain is emitted; a weak one is flagged, and one with no usable reference at all is recorded as /// `unresolved` rather than dropped. fn derive_offsets( vfuncs: &[VtFunc], hops: &HashMap<&str, Vec>>>, per_build: &[BuildVtables], target_idx: usize, gd: &mut model::Gamedata, ) -> (u32, Vec) { let mut off_ok = 0; let mut off_flag: Vec = Vec::new(); for f in vfuncs { let hv = &hops[f.class.as_str()]; let anchors = spread_anchors(per_build, target_idx, |bv| { let o0 = offset_at(&f.timeline, &bv.date)? as usize; (o0 < bv.fps.get(&f.class)?.len()).then_some(o0) }); if anchors.is_empty() { // Nothing to chain from: the offset exceeds the vtable (a struct FIELD offset like // m_iHitGroup), the class has no locatable vtable, or — the common case for a fresh // contribution — the value is dated at or after the newest reference build. We deliberately // do NOT carry the last known value forward: CS2 inherits Source-1 class NAMES (CAK47, // CBaseButton) whose LAYOUTS differ, so an ecosystem date-src offset for a coincident name is // a different game's value and would ship a wrong offset. Drop it, but VISIBLY — record it so // `unresolved` is the complete catalogue picture the consumer contract claims, and a // contributor gets a reason instead of nothing. off_flag.push(model::Flagged::new( f.name.clone(), model::FlagReason::Unresolved, "no chainable anchor (offset exceeds the vtable, class has no vtable, or the value \ post-dates every reference build)", )); continue; } // Recency-weighted vote (see chain_and_vote): only a high-confidence chain reaches the shipped // gamedata. A low-confidence chain over a cross-game catalogue is usually a coincident class NAME // (CS2 shares CAK47/CBaseEntity with GoldSrc) whose offset chained by luck — flag, never emit. match chain_and_vote(&anchors, hv, target_idx) { Some((pred, conf)) if conf >= 80 => { gd.set_offset(f.name.clone(), pred as i64); // The class whose vtable this slot indexes — half the locator, since an index alone locates // nothing. It is `class_of(f.name)`: `vtable_offset_timelines` builds `VtFunc::class` that // way and both `hops` and `bv.fps` are keyed by it, so the class the chain walked and the // class in the name are one fact, not two. Emitting it saves the consumer a name split; it // does not add information the name lacks. gd.set_class(f.name.clone(), f.class.clone()); off_ok += 1; } Some((pred, conf)) => { off_flag.push(model::Flagged::new( f.name.clone(), model::FlagReason::OffsetLowConf, format!("offset {pred}, conf {conf}% — not emitted"), )); } None => off_flag.push(model::Flagged::new( f.name.clone(), model::FlagReason::Unresolved, "no vote", )), } } (off_ok, off_flag) } /// For each class, its vtable slot-fingerprints + slot addresses from the FIRST library (in the given /// precedence order) whose primary vtable holds it — the deterministic, correctness-critical "first lib /// holding the class wins" rule. Extracted so the model and binary paths of /// `build_per_build_vtables` implement that precedence exactly once. fn target_vtables<'a>( prof: &GameProfile, imgs: impl IntoIterator, classes: &[String], ) -> ( HashMap>>>, HashMap>, ) { let imgs: Vec<&CodeImage> = imgs.into_iter().collect(); let (mut fps, mut addrs) = (HashMap::new(), HashMap::new()); for c in classes { for img in &imgs { if let Some(vt) = rtti::find_vtable(img, c, prof.max_vtable_slots) { fps.insert(c.clone(), slot_fingerprints(img, &vt.slots)); addrs.insert(c.clone(), vt.slots); break; } } } (fps, addrs) } /// Per-build vtable data (fingerprints + slot addresses + resolved anchors) plus the target's index — /// the input to both offset chaining and vtable-anchoring recovery. From the MODEL (only the target /// binary is read; past builds are synthesised from the distilled slot counts + anchor slots so the /// downstream is byte-identical) or by fingerprinting every reference build's binaries. fn build_per_build_vtables( prof: &GameProfile, cmodel: Option<&CorpusModel>, ref_builds: Vec<(String, PathBuf)>, target: &Path, classes: &[String], virt_funcs: &[(&Func, String)], ) -> (Vec, usize) { if let Some(m) = cmodel { // The one build we DO read — the target's real vtable slots + fingerprints. Load the WHOLE // profile lib set rather than reusing `images`: that map is preloaded from the libraries the // SIGNATURE half names, and the catalogue's `library` hint is community-sourced and routinely // wrong about which module a class lives in (`CSoundSystem` and `CVPhys2World` are both // catalogued "server" while their vtables are in `libsoundsystem` / `libvphysics2`). Searching // only the sig-named subset silently dropped those offsets from every release built on this // path while the `--corpus` path shipped them; the two paths must not disagree. The binary // branch below already loads the full set for exactly this reason. // // `load_build_images` yields the libs in `prof.libs` precedence order, which is what keeps the // choice deterministic: a class whose vtable appears in more than one lib must resolve to the // same image here as everywhere else, and the first lib holding it wins. let timgs = load_build_images(prof, target); let (tf, ta) = target_vtables(prof, &timgs, classes); // The target's own (real, newest) date — offset_at(timeline, target_date) then returns the latest // timeline entry (the target is newer than every corpus build by the forward-only guard); the // target's position is tracked explicitly by `target_idx` below. let target_bv = BuildVtables { date: build_date(&label_of(target)), fps: tf, addrs: ta, resolved: HashMap::new(), // synthetic: reconstructed from the model, not loaded from images, so there is nothing to report ..Default::default() }; // past builds: encode the model's distilled slot counts + anchor slots as a synthetic // BuildVtables so the downstream is byte-identical — fps length = slot_count (bounds checks), // addrs = slot indices, resolved = the anchor slot (so `position(resolved in addrs)` recovers it). let mut pb: Vec = m .builds .iter() .enumerate() .map(|(i, bm)| { let (mut fps, mut addrs) = (HashMap::new(), HashMap::new()); for c in classes { let n = m .slot_counts .get(c) .and_then(|v| v.get(i)) .copied() .unwrap_or(0); if n > 0 { fps.insert(c.clone(), vec![None; n]); addrs.insert(c.clone(), (0..n as u64).collect()); } } let mut resolved = HashMap::new(); for (f, _) in virt_funcs { if let Some(slot) = m .resolved_slot .get(&f.name) .and_then(|v| v.get(i)) .and_then(|s| s.get()) { resolved.insert(f.name.clone(), slot as u64); } } BuildVtables { date: bm.date.clone(), fps, addrs, resolved, ..Default::default() // synthetic (model-derived): no images were loaded } }) .collect(); pb.push(target_bv); let ti = pb.len() - 1; (pb, ti) } else { let mut builds = ref_builds; // the target's real (newest) date — see the model branch above; `ti` is its explicit position marker. builds.push((build_date(&label_of(target)), target.to_path_buf())); let ti = builds.len() - 1; let pb = parallel_map(&builds, default_threads(None), |(date, dir)| { let imgs = load_build_images(prof, dir); let (fps, addrs) = target_vtables(prof, &imgs, classes); let mut resolved = HashMap::new(); for (f, _) in virt_funcs { if let Some(a) = resolve_unique(f, &imgs) { resolved.insert(f.name.clone(), a); } } BuildVtables { date: date.clone(), fps, addrs, resolved, ..Default::default() // built from already-loaded images; load reporting happens there } }); (pb, ti) } } /// Vtable-slot alignment hops per class: the model ships the past consecutive-build hops and the final /// hop (latest model build -> target) is computed from the target's real vtable fingerprints; the binary /// path aligns every consecutive pair directly. Keys borrow `classes`. fn build_hops<'a>( cmodel: Option<&CorpusModel>, per_build: &[BuildVtables], target_idx: usize, classes: &'a [String], ) -> HashMap<&'a str, Vec>>> { classes .iter() .map(|c| { let hv = if let Some(m) = cmodel { let mut h: Vec>> = m .hops .get(c) .map(|v| v.iter().map(|inner| inner.to_options()).collect()) .unwrap_or_default(); let fh = match (m.latest_vtable_fps.get(c), per_build[target_idx].fps.get(c)) { (Some(a), Some(b)) => nw_align(a, b), _ => Vec::new(), }; h.push(fh); h } else { per_build .windows(2) .map(|w| match (w[0].fps.get(c), w[1].fps.get(c)) { (Some(a), Some(b)) => nw_align(a, b), _ => Vec::new(), }) .collect() }; (c.as_str(), hv) }) .collect() } /// Recover drifted VIRTUAL functions by vtable slot — content-INDEPENDENT, so it works even when the /// function changed so much no fingerprint matches. For each candidate: find builds where its signature /// still resolved *to a slot of its class vtable* (that slot index is its offset, and requiring a real slot /// auto-rejects decoy matches), chain the offset forward to the target through the order-preserving vtable /// alignments, vote on the target slot, then read `vtable[offset]` in the target and emit a sig there. /// Consumes the shared vtable pass. Returns each recovered name with the LIBRARY and address it was /// relocated to, so the caller can re-measure the footprint there — see `recover_by_string_anchor`. fn recover_virtual_sigs( prof: &GameProfile, virt_funcs: &[(&Func, String)], per_build: &[BuildVtables], hops: &HashMap<&str, Vec>>>, target_idx: usize, target_images: &HashMap, gd: &mut model::Gamedata, ) -> Vec<(String, &'static str, u64)> { let mut recovered = Vec::new(); for (f, class) in virt_funcs { let Some(hv) = hops.get(class.as_str()) else { continue; }; // anchors: time-spread builds where the sig resolved to a slot of this class's vtable let anchors = spread_anchors(per_build, target_idx, |bv| { let &addr = bv.resolved.get(&f.name)?; bv.addrs .get(class.as_str())? .iter() .position(|&a| a == addr) }); if anchors.is_empty() { continue; // never resolved to a slot -> can't anchor } // chain each anchor to the target and recency-weight-vote the target slot let Some((off, conf)) = chain_and_vote(&anchors, hv, target_idx) else { continue; }; if conf < 80 { continue; // low agreement -> leave flagged } // read the located slot in the target and emit a sig there. All CS2 .so link at vaddr 0, so // is_code(addr) is ambiguous across libraries — resolve the class's OWN vtable image // (find_vtable is class-specific) so make_sig reads the right bytes, deterministically, and // the emitted library is correct rather than a hardcoded "server". if let Some(&addr) = per_build[target_idx] .addrs .get(class.as_str()) .and_then(|s| s.get(off)) { let lib = prof.libs.iter().copied().find(|&l| { target_images .get(l) .and_then(|im| rtti::find_vtable(im, class, prof.max_vtable_slots)) .is_some_and(|vt| vt.slots.get(off) == Some(&addr)) }); if let Some(lib) = lib && let Some(sig) = emit::make_sig(&target_images[lib], addr, CORE_SIG_CAP) { gd.set_signature(f.name.clone(), lib_name_from_file(lib), sig); recovered.push((f.name.clone(), lib, addr)); } } } recovered } // ══════════════════════════════════════════════════════════════════════════════════════════════ // §8 · BACKFILL (cross-build name/offset timelines) // ══════════════════════════════════════════════════════════════════════════════════════════════ #[derive(Deserialize)] struct BackfillItem { name: String, #[serde(default)] tier: String, #[serde(default)] anchor: Option, // a distinctive string it references (drives the binary string-anchor half) #[serde(default)] class: Option, // drives the model-hops half (with slot) #[serde(default)] slot: Option, } /// Chain an offset's vtable slot BACKWARD through a class's per-build-pair alignment hops, from its known /// latest-build slot to every earlier build — the model-only history for a vtable method (no binaries). /// `hops[i]` aligns build i -> i+1 (indexed by OLD slot); inverting each hop walks the slot back in time. fn hops_timeline( hops: &[Vec>], latest_slot: usize, latest_count: usize, ) -> Vec> { let n = hops.len() + 1; let mut tl = vec![None; n]; // Guard the tip like `vtable_slot_timeline`: a slot beyond the class's latest-build vtable is NOT located // (seed None), so an out-of-range slot doesn't count as "located at the newest build" and spuriously // inflate this method's coverage in the cov() tiebreak against the bounds-checked binary method. tl[n - 1] = (latest_slot < latest_count).then_some(latest_slot); for i in (0..hops.len()).rev() { if let Some(target) = tl[i + 1] { tl[i] = hops[i].iter().position(|&m| m == Some(target)); } } tl } /// The class's per-build vtable-slot fingerprints (None where the class is absent in that build). Chained /// to walk one latest-build slot back through time. type VtSeq = Vec>>>>; /// Track a class's vtable slot BACKWARD across the raw builds by chaining the order-preserving vtable /// alignments (the `chain` logic, for one latest-build slot). Reliable where model `hops` aren't /// available — adjacent-build alignment is near-perfect — and it covers ANY class, not just model-tracked /// ones. This is the binary path for offset back-fill (the model-hops path is its binaries-free CI twin). fn vtable_slot_timeline(seq: &VtSeq, tip_slot: usize) -> Vec> { let n = seq.len(); let mut tl = vec![None; n]; if n == 0 { return tl; } tl[n - 1] = seq[n - 1] .as_ref() .filter(|f| tip_slot < f.len()) .map(|_| tip_slot); for i in (0..n - 1).rev() { if let (Some(target), Some(from), Some(to)) = (tl[i + 1], seq[i].as_ref(), seq[i + 1].as_ref()) { let map = nw_align(from, to); // from(build i) -> to(build i+1) tl[i] = map.iter().position(|&m| m == Some(target)); } } tl } /// One raw build's back-fill signals: the unique anchor address per name + each needed class's vtable /// slot fingerprints (for the vtable-chain method). struct BuildBackfill { anchor: Vec>, vt: HashMap>>>, } /// Which timeline won for a back-filled name — a closed domain that also encodes how its values read: `Anchor` /// timelines are ADDRESSES (formatted `{:#x}`), `Vtable`/`Hops` are vtable SLOTS (plain). #[derive(Clone, Copy)] enum BackfillMethod { Anchor, Vtable, Hops, } impl BackfillMethod { /// The report label. fn as_str(self) -> &'static str { match self { BackfillMethod::Anchor => "anchor", BackfillMethod::Vtable => "vtable", BackfillMethod::Hops => "hops", } } /// Timeline values are addresses (`{:#x}`), not vtable slots. fn is_address(self) -> bool { matches!(self, BackfillMethod::Anchor) } } /// Back-fill cross-build history for extrapolated (T3) names — the historical-enrichment + community-PR /// ingestion tool. An OFFSET name rides the model's `hops` (no binaries — the vtable-method PR path); a /// SIG / self-named name rides the string-anchor locator over the raw builds. Reports per-name history /// depth + consistency-since-first-appearance: the measure of how many T3 names graduate to first-class /// (deep + consistent history == high confidence, regardless of T3 origin). The compact per-name history /// it emits is the persistent timeline that survives deleting the binaries. pub fn backfill_cmd( prof: &GameProfile, corpus: Option<&Path>, corpus_model: Option<&Path>, lib: &str, names: &Path, threads: Option, out: Option<&Path>, ) -> Result<()> { // A backfill needs a build source to resolve anchors against; with neither it would run to completion // reporting every name as "not located" and exit 0 — a vacuous success. Fail loudly, as `produce` does. ensure!( corpus.is_some() || corpus_model.is_some(), "backfill needs --corpus or --corpus-model — with neither there is nothing \ to resolve anchors against" ); let items: Vec = serde_json::from_str(&std::fs::read_to_string(names)?) .context("parse backfill names json")?; let model: Option = match corpus_model { Some(p) => { Some(serde_json::from_str(&std::fs::read_to_string(p)?).context("parse corpus model")?) } None => None, }; let model_dates: Vec = model .as_ref() .map(|m| m.builds.iter().map(|b| b.date.clone()).collect()) .unwrap_or_default(); // The binary half: per raw build, locate each name by its anchor string AND fingerprint each needed // class's vtable (for the vtable-chain method — the reliable, all-classes offset back-fill). let builds = match corpus { Some(c) => find_builds(prof, c)?, None => Vec::new(), }; let anchor_dates: Vec = builds.iter().map(|d| build_date(&label_of(d))).collect(); let vt_needed: BTreeSet = items .iter() .filter(|it| it.slot.is_some_and(|s| s >= 0)) .filter_map(|it| it.class.clone()) .collect(); let per_build: Vec = if builds.is_empty() { Vec::new() } else { parallel_map(&builds, default_threads(threads), |dir| { let empty = BuildBackfill { anchor: vec![None; items.len()], vt: HashMap::new(), }; let Some(img) = find_file(dir, lib, 8).and_then(|p| CodeImage::load(&p).ok()) else { return empty; }; let idx = xref::XrefIndex::build(&img); let anchor = items .iter() .map(|it| { it.anchor.as_deref().and_then(|a| { match xref::funcs_using_string(&img, &idx, a).as_slice() { [addr] => Some(*addr), _ => None, // absent or ambiguous in this build } }) }) .collect(); // one RTTI sweep, keep the needed primary vtables' slot fingerprints let mut vt = HashMap::new(); if !vt_needed.is_empty() { for cv in rtti::enumerate_vtables(&img, prof.max_vtable_slots) .into_iter() .filter(|c| c.offset_to_top == 0 && vt_needed.contains(&c.name)) { let fps = slot_fingerprints(&img, &cv.slots); vt.insert(cv.name, fps); } } BuildBackfill { anchor, vt } }) }; let mut report = Vec::new(); let mut buckets = [0usize; 4]; // deep-first-class, recent-consistent, flaky, not-located let cov = |tl: &[Option]| tl.iter().filter(|x| x.is_some()).count(); for (i, it) in items.iter().enumerate() { // Compute every applicable timeline, then KEEP the one with the most cross-build coverage — so a // self-named offset that anchors perfectly isn't regressed onto a weaker vtable chain, and a // dict-exact offset with no self-string is rescued by the vtable chain. let mut best: (BackfillMethod, &[String], Vec>) = ( BackfillMethod::Anchor, &anchor_dates, (0..per_build.len()) .map(|b| per_build[b].anchor[i]) .collect(), ); // vtable-chain over binaries (offset, any class the RTTI sweep captured) if let (Some(cls), Some(slot)) = (it.class.as_deref(), it.slot.filter(|&s| s >= 0)) { if per_build.iter().any(|b| b.vt.contains_key(cls)) { let seq: VtSeq = per_build.iter().map(|b| b.vt.get(cls).cloned()).collect(); let tl: Vec> = vtable_slot_timeline(&seq, slot as usize) .into_iter() .map(|o| o.map(|s| s as u64)) .collect(); if cov(&tl) > cov(&best.2) { best = (BackfillMethod::Vtable, &anchor_dates, tl); } } // model hops (binaries-free) — the CI twin; used if it beats the binary methods if let Some(h) = model .as_ref() .and_then(|m| m.hops.get(cls)) .filter(|h| !h.is_empty()) { let h: Vec>> = h.iter().map(|inner| inner.to_options()).collect(); let latest_count = model .as_ref() .and_then(|m| m.slot_counts.get(cls)) .and_then(|v| v.last()) .copied() .unwrap_or(0); let tl: Vec> = hops_timeline(&h, slot as usize, latest_count) .into_iter() .map(|o| o.map(|s| s as u64)) .collect(); if cov(&tl) > cov(&best.2) { best = (BackfillMethod::Hops, &model_dates, tl); } } } let (method, dates, timeline) = best; let n = dates.len(); let located: Vec = (0..n).filter(|&b| timeline[b].is_some()).collect(); let count = located.len(); let first = located.first().copied(); let span = first.map_or(0, |f| n - f); let consistency = if span > 0 { count as f64 / span as f64 } else { 0.0 }; let bucket = if count == 0 { 3 } else if count * 2 >= n && consistency >= 0.9 { 0 // deep, consistent history == first-class regardless of T3 origin } else if consistency >= 0.9 { 1 // recent but consistent since it appeared } else { 2 // flaky — the anchor drifts / isn't distinctive }; buckets[bucket] += 1; // compact history: only located builds -> value (vtable slot for hops, address for anchor). let history: serde_json::Map = located .iter() .filter_map(|&b| { timeline[b].map(|v| { // hops/vtable timelines are vtable SLOTS; anchor is an ADDRESS. let val = if method.is_address() { json!(format!("{v:#x}")) } else { json!(v) }; (dates[b].clone(), val) }) }) .collect(); report.push(json!({ "name": it.name, "tier": it.tier, "method": method.as_str(), "builds_located": count, "of_builds": n, "first_build": first.map(|f| dates[f].clone()), "consistency": (consistency * 100.0).round() / 100.0, "history": history, })); } eprintln!( "backfill: {} names over {} corpus builds / {} model builds", items.len(), anchor_dates.len(), model_dates.len() ); eprintln!( " graduates: {} DEEP first-class (>=50% coverage, >=0.9 consistent), {} recent-but-consistent, {} flaky, {} not located", buckets[0], buckets[1], buckets[2], buckets[3] ); let doc = json!({ "names": items.len(), "corpus_builds": anchor_dates.len(), "model_builds": model_dates.len(), "report": report, }); match out { Some(p) => { std::fs::write(p, serde_json::to_string(&doc)?) .with_context(|| format!("write {}", p.display()))?; eprintln!(" -> {}", p.display()); } None => println!("{}", serde_json::to_string_pretty(&doc)?), } Ok(()) } #[cfg(test)] mod tests { use super::*; // ---- the identity check's key form. This guard rejects an address the binary itself says is a // different function, and it is keyed by SHORT library name. A caller holding the FILE name made // `get` miss, `?` return, and the whole check pass silently — which is indistinguishable from "no // contradiction found". These pin both halves: that it fires, and that the wrong form is loud. ---- fn identity_fixture() -> Identity { Identity(HashMap::from([( "server".to_string(), LibIdentity { // Valve's registry says this address is `CLogicRelay::Trigger`, not what we asked about. vscript_at: HashMap::from([(0x1000, "Trigger".to_string())]), class_size: HashMap::from([("CBaseEntity".to_string(), 0x100)]), }, )])) } // mov %rdi,%r13 ; cmpb $0,0x7bc(%r13) ; ret — reaches `this+0x7bc`, past a 0x100-byte class. const REACHES_PAST: &[u8] = &[ 0x49, 0x89, 0xFD, 0x41, 0x80, 0xBD, 0xBC, 0x07, 0x00, 0x00, 0x00, 0xC3, ]; #[test] fn a_contradiction_is_found_under_the_short_library_name() { let img = crate::elf::CodeImage::for_test(0x1000, REACHES_PAST); let why = identity_fixture() .contradiction("server", &img, 0x1000, "CBaseEntity::SetAbsOrigin") .expect("registry disagrees AND the reach exceeds the class — both halves hold"); assert!(why.contains("Trigger"), "{why}"); } #[test] #[should_panic(expected = "keyed by SHORT library name")] fn the_file_name_form_is_loud_rather_than_a_silent_pass() { let img = crate::elf::CodeImage::for_test(0x1000, REACHES_PAST); // The same contradiction, looked up by the form `images` is keyed by. Before this was asserted it // returned `None` — "no contradiction" — on every call. identity_fixture().contradiction("libserver.so", &img, 0x1000, "CBaseEntity::SetAbsOrigin"); } // ---- contribution intake gate (the fork-facing surface) ---- #[test] fn is_iso_date_accepts_only_yyyy_mm_dd() { assert!(is_iso_date("2026-07-23")); assert!(is_iso_date("2015-01-01")); // wrong shape / separators / length / non-digit all rejected — these would otherwise slip past // intake and only fail deep in the timeline builder's `src[4]==b'-'` check. assert!(!is_iso_date("2026-7-3")); // unpadded assert!(!is_iso_date("07/23/2026")); // wrong separator assert!(!is_iso_date("2026-07-23T00:00")); // trailing time assert!(!is_iso_date("2026_07_23")); // underscores assert!(!is_iso_date("not-a-date")); // non-digit in digit slots assert!(!is_iso_date("")); // empty } // ---- hops identity-compression: an identity alignment is `Identity(len)` IN MEMORY and one integer on // the wire; a shift is `Explicit` and a full array. Both forms round-trip, and a pre-compression model // (identity stored as a full array) must load to the identical `Identity` Hop. ---- #[test] fn hop_serde_compresses_identity_and_round_trips() { let s = |o: Option| Slot::from_opt(o); // identity/empty alignment -> the Identity variant -> a single integer on the wire. let ident = Hop::from_slots(vec![s(Some(0)), s(Some(1)), s(Some(2))]); assert!(matches!(ident, Hop::Identity(3))); assert_eq!(serde_json::to_string(&ident).unwrap(), "3"); assert!(matches!(Hop::from_slots(vec![]), Hop::Identity(0))); assert_eq!(serde_json::to_string(&Hop::default()).unwrap(), "0"); // any non-identity alignment (a drop, or a reorder) -> Explicit -> the explicit array, nulls for drops. let dropped = Hop::from_slots(vec![s(Some(0)), s(None), s(Some(2))]); assert!(matches!(dropped, Hop::Explicit(_))); assert_eq!(serde_json::to_string(&dropped).unwrap(), "[0,null,2]"); // every form round-trips (Hop derives PartialEq; Slot has no Debug so assert! not assert_eq!). for slots in [ vec![s(Some(0)), s(Some(1)), s(Some(2))], // identity vec![], // empty vec![s(Some(0)), s(None), s(Some(2))], // dropped slot vec![s(Some(1)), s(Some(0))], // a real reorder = non-identity ] { let h = Hop::from_slots(slots); let j = serde_json::to_string(&h).unwrap(); let back: Hop = serde_json::from_str(&j).unwrap(); assert!(back == h, "round-trip {j}"); } // backward-compat: an identity alignment written as a full array normalizes to the identical // Identity(3) the compact integer produces. let from_old_array: Hop = serde_json::from_str("[0,1,2]").unwrap(); let from_new_int: Hop = serde_json::from_str("3").unwrap(); assert!(from_old_array == from_new_int); assert!(matches!(from_old_array, Hop::Identity(3))); } // ---- the CS# text the live oracle re-parses round-trips back to the same locators ---- // render_monolith_cssharp writes the string; run_live_oracle strips comments and reads it via // read_gamedata_str. A drift between writer and reader would silently change what gets validated, so // pin that the core (sig) + high_confidence (offset) locators survive the round trip byte-for-byte. #[test] fn cssharp_render_round_trips_through_read_gamedata_str() { use model::{Counts, Entry, MonoEntry, MonoMeta, Monolith, Provenance, Tier}; let mono = Monolith { meta: MonoMeta { game_key: "csgo".into(), game: "Counter-Strike 2".into(), source_build: "test".into(), version: "t-1-0".into(), counts: Counts { core: 1, high_confidence: 1, experimental: 0, unresolved: 0, }, alias_groups: 0, aliased_names: 0, }, core: BTreeMap::from([( "CBaseEntity::TakeDamage".to_string(), MonoEntry { locator: Entry::signature("server", "48 8B 05 ? ? ? ?"), abi: None, provenance: Provenance { source: Some("catalogue".into()), ..Provenance::with_tier(Tier::Core) }, validated: None, aliases: Vec::new(), }, )]), high_confidence: BTreeMap::from([( "CCSPlayerPawn::IsBot".to_string(), MonoEntry { locator: Entry::offset(42), abi: None, provenance: Provenance::with_tier(Tier::SelfNamed), validated: None, aliases: Vec::new(), }, )]), experimental: BTreeMap::new(), unresolved: BTreeMap::new(), }; let text = render::render_monolith_cssharp(&mono, model::TierSelect::HighConfidence); let gd = read_gamedata_str(&text).expect("rendered cssharp must parse back"); let sig = render::entry_from_value(&gd["CBaseEntity::TakeDamage"]); assert_eq!( sig.signature.map(|s| s.linux).as_deref(), Some("48 8B 05 ? ? ? ?") ); let off = render::entry_from_value(&gd["CCSPlayerPawn::IsBot"]); assert_eq!(off.offset, Some(42)); } // ---- link_aliases groups by LOCATOR IDENTITY, across tiers, and refuses to group a bare slot. The // last of those is the one worth pinning: an `offset` with no `class` names no vtable, so bucketing // those by slot index would put every unbound "slot 0" in one group and assert 1,080 CS2 names are // aliases of each other. ---- #[test] fn link_aliases_groups_by_locator_and_never_by_a_bare_slot() { use model::{Entry, MonoEntry, Provenance, Tier}; let e = |loc: Entry| MonoEntry { locator: loc, abi: None, provenance: Provenance::with_tier(Tier::Core), validated: None, aliases: Vec::new(), }; let sig = |lib: &str, pat: &str| e(Entry::signature(lib, pat)); let slot = |class: Option<&str>, n: i64| { e(Entry { class: class.map(str::to_string), ..Entry::offset(n) }) }; let mut core: BTreeMap = BTreeMap::from([ ("A::Make".into(), sig("server", "48 8B 05")), ("Make".into(), sig("server", "48 8B 05")), // same pattern, DIFFERENT library — a different function, so not a group. ("Other::Make".into(), sig("engine2", "48 8B 05")), // bare slots: identical index, no class, so nothing links them. ("IdleState::OnEnter".into(), slot(None, 0)), ("HideState::OnEnter".into(), slot(None, 0)), ]); let mut high: BTreeMap = BTreeMap::from([ // cross-tier: joins the `core` pair above, which is the pairing a reader is least likely to spot. ("UTIL::Make".into(), sig("server", "48 8B 05")), ("CFoo::Tick".into(), slot(Some("CFoo"), 12)), ("CFoo::Update".into(), slot(Some("CFoo"), 12)), ("CBar::Tick".into(), slot(Some("CBar"), 12)), ]); let (groups, covered) = link_aliases(&mut core, &mut high); assert_eq!((groups, covered), (2, 5)); // the 3-name sig group + the 2-name CFoo group // Cross-tier membership, key-sorted, and never self-referential. assert_eq!(core["A::Make"].aliases, ["Make", "UTIL::Make"]); assert_eq!(high["UTIL::Make"].aliases, ["A::Make", "Make"]); // A shared pattern in another library is a different address. assert!(core["Other::Make"].aliases.is_empty()); // A class-bound slot groups; the same slot on another class does not join it. assert_eq!(high["CFoo::Tick"].aliases, ["CFoo::Update"]); assert!(high["CBar::Tick"].aliases.is_empty()); // The whole point: two bare slot-0 entries are NOT claimed to be the same function. assert!(core["IdleState::OnEnter"].aliases.is_empty()); assert!(core["HideState::OnEnter"].aliases.is_empty()); } // ---- annotate_validation must NOT claim validation it didn't perform ("never lies"). A live-confirmed // entry -> Some(true); an entry the oracle KEPT-BUT-COULD-NOT-CHECK (verdict None) -> None (unverified, // NOT true); an entry the oracle dropped (absent from the verdict map) -> Some(false). ---- #[test] fn annotate_validation_does_not_overclaim() { use model::{Counts, Entry, MonoEntry, MonoMeta, Monolith, Provenance, Tier}; let entry = |loc: Entry, tier| MonoEntry { locator: loc, abi: None, provenance: Provenance::with_tier(tier), validated: None, aliases: Vec::new(), }; let mut mono = Monolith { meta: MonoMeta { game_key: "csgo".into(), game: "Counter-Strike 2".into(), source_build: "test".into(), version: "t-1-0".into(), counts: Counts { core: 2, high_confidence: 1, experimental: 0, unresolved: 0, }, alias_groups: 0, aliased_names: 0, }, core: BTreeMap::from([ ( "Live::Fn".to_string(), entry(Entry::signature("server", "48 8B"), Tier::Core), ), ( "Unchecked::Fn".to_string(), entry(Entry::signature("vscript", "55 48"), Tier::Core), ), ]), high_confidence: BTreeMap::from([( "Dropped::Fn".to_string(), entry(Entry::offset(42), Tier::SelfNamed), )]), experimental: BTreeMap::new(), unresolved: BTreeMap::new(), }; // Live::Fn was live-confirmed; Unchecked::Fn was kept but its lib wasn't mapped (verdict None); // Dropped::Fn is ABSENT from the map (the oracle dropped it). let verdicts = BTreeMap::from([ ("Live::Fn".to_string(), Some(true)), ("Unchecked::Fn".to_string(), None), ]); annotate_validation(&mut mono, &verdicts); assert_eq!(mono.core["Live::Fn"].validated, Some(true)); // confirmed -> true assert_eq!(mono.core["Unchecked::Fn"].validated, None); // kept-unverified -> NOT true, never claimed assert_eq!(mono.high_confidence["Dropped::Fn"].validated, Some(false)); // dropped -> false } }