5680 lines
266 KiB
Rust
5680 lines
266 KiB
Rust
//! The OFFLINE derivation engine: catalogue resolution, corpus-model distillation, gamedata folding,
|
||
//! the experimental band, and monolith assembly. Everything here is a pure function of files on disk —
|
||
//! **nothing in this module attaches to a running process.** Live validation, the typed schema walk and
|
||
//! change classification are `produce`'s, and the split is the design rather than an accident: it is what
|
||
//! lets an offline build produce a smaller honest artifact with no server anywhere in sight.
|
||
//!
|
||
//! The `source2rosetta` binary is a thin clap front-end over this module and `produce` (see `main.rs`);
|
||
//! keeping the engine in the library lets CI and tests link and call it directly instead of shelling out.
|
||
|
||
use crate::elf::CodeImage;
|
||
use crate::locate::{find_file, load_lib};
|
||
use crate::par::{default_threads, parallel_map};
|
||
use crate::profile::GameProfile;
|
||
use crate::sig::Pattern;
|
||
use crate::taxonomy::{
|
||
FullName, class_of, clean_offset_class, guess_tier, is_dead_weight_class, is_dead_weight_name,
|
||
is_serializer_plumbing, parse_ret, protobuf_message_classes,
|
||
};
|
||
use crate::{abi, concmd, emit, fingerprint, model, pulse, render, rtti, schema, valvetab, xref};
|
||
use anyhow::{Context, Result, bail, ensure};
|
||
use serde::Deserialize;
|
||
use serde_json::{Value, json};
|
||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||
use std::path::{Path, PathBuf};
|
||
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// TABLE OF CONTENTS (grep `§N ·` to jump)
|
||
// §1 · Catalogue model & shared vocabulary §5 · Monolith assembly
|
||
// §2 · Catalogue & image loading §6 · Corpus-model distillation
|
||
// §3 · Fingerprint alignment §7 · Gamedata derivation (catalogue → sigs/offsets)
|
||
// §4 · Gamedata fold & experimental band §8 · Backfill (cross-build timelines)
|
||
// This module is the OFFLINE engine — nothing here attaches to a running server. Companion modules:
|
||
// `taxonomy` (name/dead-weight predicates), `schema` (offline reader + live type walk), `produce`
|
||
// (CI orchestration + the whole LIVE verification subsystem — oracle / validate-verify / fuzzer / launch).
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// §1 · CATALOGUE MODEL & SHARED VOCABULARY
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
|
||
/// class name -> its vtable slot fingerprints (index = slot; `None` where a slot didn't decode).
|
||
/// The recurring shape behind every cross-build vtable alignment.
|
||
type VtableFps = HashMap<String, Vec<Option<Vec<u32>>>>;
|
||
|
||
/// Candidate target addresses for one function, each tagged with how many distinct era-sigs voted
|
||
/// for it (most-voted first).
|
||
type VoteList = Vec<(u64, usize)>;
|
||
|
||
/// Where `gamedata` pulls its cross-build reference signals from — exactly one of the two.
|
||
#[derive(Clone, Copy)]
|
||
pub(crate) enum CorpusSource<'a> {
|
||
/// The raw corpus of build binaries (each fingerprinted on the fly).
|
||
Binaries(&'a Path),
|
||
/// A distilled corpus model (already parsed) — then only the target binary is read. The caller owns the
|
||
/// model so it can be parsed ONCE and reused (`produce` derives through this borrow, then hands the same
|
||
/// model by value to the sidecar fold — the ~571 MB Dota model is parsed a single time, not twice).
|
||
Model(&'a CorpusModel),
|
||
}
|
||
|
||
#[derive(Deserialize)]
|
||
struct Func {
|
||
name: String,
|
||
#[serde(default)]
|
||
library: Vec<String>,
|
||
#[serde(default)]
|
||
variants: Vec<Variant>,
|
||
}
|
||
|
||
/// A catalogue variant's locator kind. `Other` (via serde) absorbs any future/unknown kind, so an
|
||
/// unexpected value is skipped rather than a hard parse error.
|
||
#[derive(Deserialize, PartialEq, Eq, Clone, Copy)]
|
||
#[serde(rename_all = "kebab-case")]
|
||
enum VariantKind {
|
||
Offset,
|
||
Signature,
|
||
StringAnchor,
|
||
VtableOffset,
|
||
#[serde(other)]
|
||
Other,
|
||
}
|
||
|
||
/// The platform a variant applies to. A locator is used for the linux derive when it is `Linux` or the
|
||
/// platform-agnostic `Any`.
|
||
#[derive(Deserialize, PartialEq, Eq, Clone, Copy, Default)]
|
||
#[serde(rename_all = "lowercase")]
|
||
enum Platform {
|
||
Linux,
|
||
Any,
|
||
Windows,
|
||
#[serde(other)]
|
||
#[default]
|
||
Other,
|
||
}
|
||
|
||
impl Platform {
|
||
fn is_linux(self) -> bool {
|
||
matches!(self, Platform::Linux | Platform::Any)
|
||
}
|
||
}
|
||
|
||
#[derive(Deserialize)]
|
||
struct Variant {
|
||
kind: VariantKind,
|
||
#[serde(default)]
|
||
platform: Platform,
|
||
value: String,
|
||
#[serde(default)]
|
||
src: String, // era tag (v1.0.N) for gamedata-sourced variants
|
||
}
|
||
|
||
/// Which classes get vtable-slot hops in a distilled corpus model. clap validates the choice at parse
|
||
/// time (and lists it in --help), so `corpus-model` can match it exhaustively — no silent fallback.
|
||
#[derive(
|
||
Clone,
|
||
Copy,
|
||
PartialEq,
|
||
Eq,
|
||
Debug,
|
||
Default,
|
||
clap::ValueEnum,
|
||
serde::Serialize,
|
||
serde::Deserialize,
|
||
)]
|
||
#[serde(rename_all = "kebab-case")]
|
||
pub enum ClassScope {
|
||
/// Every real game class (the default) — enough for any modding offset to derive model-only.
|
||
#[default]
|
||
Clean,
|
||
/// Also template / protobuf / NetworkVar junk.
|
||
All,
|
||
/// Only the classes the catalogue names.
|
||
Catalogue,
|
||
}
|
||
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// §2 · CATALOGUE & IMAGE LOADING
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
|
||
pub(crate) fn lib_filename(prof: &GameProfile, lib: &str) -> String {
|
||
match lib {
|
||
"server" => prof.server_lib.into(),
|
||
"engine2" => prof.engine_lib.into(),
|
||
other => format!("lib{other}.so"),
|
||
}
|
||
}
|
||
|
||
fn lib_name_from_file(fname: &str) -> String {
|
||
fname
|
||
.strip_prefix("lib")
|
||
.unwrap_or(fname)
|
||
.strip_suffix(".so")
|
||
.unwrap_or(fname)
|
||
.to_string()
|
||
}
|
||
|
||
fn candidate_libs(prof: &GameProfile, f: &Func) -> Vec<String> {
|
||
let named: Vec<&String> = f.library.iter().filter(|l| *l != "unknown").collect();
|
||
if named.is_empty() {
|
||
vec![prof.server_lib.into(), prof.engine_lib.into()]
|
||
} else {
|
||
named.iter().map(|l| lib_filename(prof, l)).collect()
|
||
}
|
||
}
|
||
|
||
/// Why no pass could produce a locator for `f` — the DETAIL that ships beside its `unresolved` entry.
|
||
///
|
||
/// Worth distinguishing rather than saying "not found", because the three cases are different claims and
|
||
/// only one of them is about this build. Most of what lands here was never a function: the catalogue is
|
||
/// harvested from other people's dumps, and a dumper's own JSON keys (`build_number`, `dwEntityList`,
|
||
/// `attack`) arrive looking exactly like names. Reporting those as functions this build failed to locate
|
||
/// would be its own kind of lie — the artifact would assert that CS2 has a function called `jump`.
|
||
fn unresolvable_because(f: &Func) -> &'static str {
|
||
let kinds: Vec<VariantKind> = f.variants.iter().map(|v| v.kind).collect();
|
||
if !kinds.is_empty() && kinds.iter().all(|k| *k == VariantKind::Offset) {
|
||
// A raw member offset, not a vtable slot — the kind `ContributionKind` deliberately excludes.
|
||
return "catalogue carries only `offset` variants — a raw member offset, which this tool does \
|
||
not derive (it derives signatures and VTABLE slots). Often not a function at all.";
|
||
}
|
||
if !linux_sigs(f).is_empty() && f.library.iter().all(|l| l == "unknown") {
|
||
return "signature variants name no library, so there is no image to scan — the harvested source \
|
||
did not record one";
|
||
}
|
||
if linux_sigs(f).is_empty() && string_anchors(f).is_empty() {
|
||
return "no linux signature, no string anchor and no vtable offset — nothing this build knows \
|
||
how to look for";
|
||
}
|
||
"no locator produced by any pass — the signature did not resolve uniquely and no string anchor \
|
||
referenced exactly one function in the target"
|
||
}
|
||
|
||
fn linux_sigs(f: &Func) -> Vec<&Variant> {
|
||
f.variants
|
||
.iter()
|
||
.filter(|v| v.kind == VariantKind::Signature && v.platform.is_linux())
|
||
.collect()
|
||
}
|
||
|
||
/// String-anchor locators for `f`: distinctive strings the function references. Stable across builds
|
||
/// even when byte sigs drift, so `gamedata` re-resolves them per build via the xref index.
|
||
fn string_anchors(f: &Func) -> Vec<&str> {
|
||
f.variants
|
||
.iter()
|
||
.filter(|v| v.kind == VariantKind::StringAnchor && v.platform.is_linux())
|
||
.map(|v| v.value.as_str())
|
||
.collect()
|
||
}
|
||
|
||
pub(crate) fn label_of(dir: &Path) -> String {
|
||
dir.file_name()
|
||
.map(|s| s.to_string_lossy().into_owned())
|
||
.unwrap_or_else(|| dir.display().to_string())
|
||
}
|
||
|
||
/// Load every library the catalogue references (usually just libserver/libengine2) from a build, plus the
|
||
/// names that were PRESENT and would not parse.
|
||
///
|
||
/// The two failures are not the same fact and must not read the same. A library the catalogue names but
|
||
/// this tree does not have is ordinary — `library` is community-sourced and routinely wrong, and
|
||
/// `seed-cs2.json` names `client`, `hammer` and `undefined` among others, none of which exist in a
|
||
/// dedicated-server tree. A library that IS there and does not parse is a broken input, and every catalogue
|
||
/// entry naming it then resolves nowhere and ships as `SigDrifted` — "no unique/recovered signature in
|
||
/// target" — asserting that a signature drifted inside a file nobody opened.
|
||
fn preload_images(
|
||
prof: &GameProfile,
|
||
funcs: &[Func],
|
||
dir: &Path,
|
||
) -> (HashMap<String, CodeImage>, Vec<String>) {
|
||
let mut wanted: BTreeSet<String> = BTreeSet::new();
|
||
for f in funcs {
|
||
if !linux_sigs(f).is_empty() {
|
||
wanted.extend(candidate_libs(prof, f));
|
||
}
|
||
}
|
||
let mut images = HashMap::new();
|
||
let mut unreadable = Vec::new();
|
||
for fname in &wanted {
|
||
let Some(p) = find_file(dir, fname, 8) else {
|
||
continue; // absent: normal, see above
|
||
};
|
||
match CodeImage::load(&p) {
|
||
Ok(img) => {
|
||
images.insert(fname.clone(), img);
|
||
}
|
||
Err(e) => unreadable.push(format!("{fname} ({e:#})")),
|
||
}
|
||
}
|
||
(images, unreadable)
|
||
}
|
||
|
||
/// Every distinct address a UNIQUE-matching, de-duplicated era-sig of `f` resolves to in `img`, each mapped
|
||
/// to how many distinct era-sigs voted for it. The shared scan behind `locate_addr` / `locate_candidates` /
|
||
/// `resolve_unique` — they differ only in how they aggregate this map (unique-key / vote-ranked / cross-image
|
||
/// union). A sig with a non-unique match (0 or >1 hits) contributes nothing; the seen-set dedups equal sigs.
|
||
fn scan_sig_hits(f: &Func, img: &CodeImage) -> BTreeMap<u64, usize> {
|
||
let mut votes: BTreeMap<u64, usize> = BTreeMap::new();
|
||
let mut seen_sig = HashSet::new();
|
||
for v in linux_sigs(f) {
|
||
if !seen_sig.insert(v.value.as_str()) {
|
||
continue;
|
||
}
|
||
if let Ok(pat) = Pattern::parse(&v.value)
|
||
&& let [addr] = img.find(&pat).as_slice()
|
||
{
|
||
*votes.entry(*addr).or_default() += 1;
|
||
}
|
||
}
|
||
votes
|
||
}
|
||
|
||
/// Two things the binary states about an address, kept per image so a name can be checked against both.
|
||
///
|
||
/// Built once per image set and shared read-only across the resolution threads. Both halves are OFFLINE —
|
||
/// the VScript registry constant-propagates out of its initialiser and the SchemaSystem states each class's
|
||
/// instance size in static data — so this check runs identically in an offline derive, a live one, and the
|
||
/// distill.
|
||
struct LibIdentity {
|
||
/// Implementation address -> the C++ name Valve's VScript registry registers there. Ground truth: it is
|
||
/// the binary naming its own function, not an inference about it.
|
||
vscript_at: HashMap<u64, String>,
|
||
/// Schema class -> instance size in bytes, as the class states it.
|
||
class_size: HashMap<String, u64>,
|
||
}
|
||
|
||
/// The identity evidence for every preloaded image, keyed by SHORT library name (`server`) — never the
|
||
/// file name it was loaded from. See [`Identity::contradiction`] for why that distinction is load-bearing.
|
||
pub(crate) struct Identity(HashMap<String, LibIdentity>);
|
||
|
||
impl Identity {
|
||
fn of(images: &HashMap<String, CodeImage>) -> Self {
|
||
Identity(
|
||
images
|
||
.iter()
|
||
.map(|(fname, img)| {
|
||
let vscript_at = crate::vscript::vscript_functions(img)
|
||
.into_iter()
|
||
.filter_map(|v| match v.imp {
|
||
// Only a plain address identifies a function here; a virtual `Slot` names a
|
||
// vtable index, which locates nothing without the class.
|
||
Some(crate::vscript::Impl::Addr(a)) if a != 0 => Some((a, v.cpp_name)),
|
||
_ => None,
|
||
})
|
||
.collect();
|
||
let class_size = schema::enumerate_schema(img)
|
||
.into_iter()
|
||
.filter(|c| c.size > 0)
|
||
.map(|c| (c.name, c.size as u64))
|
||
.collect();
|
||
(
|
||
lib_name_from_file(fname),
|
||
LibIdentity {
|
||
vscript_at,
|
||
class_size,
|
||
},
|
||
)
|
||
})
|
||
.collect(),
|
||
)
|
||
}
|
||
|
||
/// Why `addr` CONTRADICTS `name`, or `None` if it does not.
|
||
///
|
||
/// **A conjunction of two independent contradictions, and it needs both.** Either alone is too noisy to
|
||
/// reject on, which is the whole reason this is shaped as an AND:
|
||
///
|
||
/// - *Valve's registry names it something else.* Alone this rejects real aliases — a dozen CS2 bindings
|
||
/// are bound STRAIGHT to the native method rather than through a script wrapper, so `SetAbsOrigin` and
|
||
/// `CBaseEntity::SetAbsOrigin` are legitimately one address, as is `ScriptSetSize` /
|
||
/// `CBaseModelEntity::SetCollisionBounds` whose names do not even resemble each other.
|
||
/// - *The code says it is a different class.* Alone this rejects working locators whose NAME merely
|
||
/// carries the wrong class prefix — measured: four `CPathMover::` entries that are really `CFuncMover`
|
||
/// setters, and two `CBasePlayerController::` entries that are really `CCSPlayerController`. Those
|
||
/// locate correctly; only their qualifier is wrong, and dropping them would lose real call sites.
|
||
///
|
||
/// Together they fire on the case where Valve names the address one thing and the machine code agrees it
|
||
/// is not the class the catalogue claims. Measured across 3,988 CS2 entries: **exactly one hit**, and it
|
||
/// was a genuine defect that had shipped in a release (`CBaseEntity::DispatchTraceAttack`, in fact
|
||
/// `CLogicRelay::Trigger`). n=1, so this rejects one entry rather than aborting a release.
|
||
fn contradiction(&self, lib: &str, img: &CodeImage, addr: u64, name: &str) -> Option<String> {
|
||
// `Identity::of` gives every loaded image an entry, and both call sites key from a file in that
|
||
// same map, so a miss here does NOT mean "a library with no evidence" — it means the caller holds
|
||
// the wrong FORM of the key, and the `?` this used to be then disabled the entire check without a
|
||
// word. That is precisely how it sat dead on the fold path.
|
||
//
|
||
// NOT a `debug_assert!`: that compiles out in release, and the shipped binary is a release build,
|
||
// so the guard would have gone on silently passing in the one configuration that matters.
|
||
let Some(lib) = self.0.get(lib) else {
|
||
panic!("Identity is keyed by SHORT library name (`server`), got `{lib}`");
|
||
};
|
||
let (class, method) = name.split_once("::")?;
|
||
let registered = lib.vscript_at.get(&addr)?;
|
||
if names_correspond(registered, method) {
|
||
return None; // Valve's name for this address agrees — a direct binding, not a contradiction
|
||
}
|
||
let size = *lib.class_size.get(class)?;
|
||
let reach = abi::this_reach(img, addr)?;
|
||
(reach >= size).then(|| {
|
||
format!(
|
||
"Valve's VScript registry registers {addr:#x} as `{registered}`, and the code reaches \
|
||
`this+{reach}` on a `{class}` of {size} bytes — the address is not this function"
|
||
)
|
||
})
|
||
}
|
||
}
|
||
|
||
/// Whether a VScript-registered C++ name and a catalogue method name are the same function under two
|
||
/// spellings. Valve prefixes script-bound wrappers inconsistently (`ScriptSetAbsAngles`, `Script_TakeDamage`,
|
||
/// bare `SetAbsOrigin`), so the prefix is stripped before comparing, and containment either way is accepted
|
||
/// because the two vocabularies abbreviate differently (`GetEHandle` / `GetRefEHandle`).
|
||
fn names_correspond(registered: &str, method: &str) -> bool {
|
||
let norm = |s: &str| {
|
||
s.trim_start_matches("Script_")
|
||
.trim_start_matches("Script")
|
||
.to_ascii_lowercase()
|
||
};
|
||
let (a, b) = (norm(registered), norm(method));
|
||
!a.is_empty() && !b.is_empty() && (a.contains(&b) || b.contains(&a))
|
||
}
|
||
|
||
/// The unique address of `f` in the preloaded images, if exactly one variant resolves cleanly.
|
||
fn locate_addr<'a>(
|
||
prof: &GameProfile,
|
||
f: &Func,
|
||
images: &'a HashMap<String, CodeImage>,
|
||
) -> Option<(&'a CodeImage, u64, String)> {
|
||
locate_addr_ident(prof, f, images, None)
|
||
}
|
||
|
||
/// `locate_addr`, refusing an address the binary itself contradicts (see [`Identity::contradiction`]).
|
||
/// Passing `None` skips the check, which is what the callers that have no image set to build it from do.
|
||
fn locate_addr_ident<'a>(
|
||
prof: &GameProfile,
|
||
f: &Func,
|
||
images: &'a HashMap<String, CodeImage>,
|
||
ident: Option<&Identity>,
|
||
) -> Option<(&'a CodeImage, u64, String)> {
|
||
let (img, fname) = candidate_libs(prof, f)
|
||
.into_iter()
|
||
.find_map(|fname| images.get(&fname).map(|img| (img, fname)))?;
|
||
let hits = scan_sig_hits(f, img);
|
||
let [addr] = hits.keys().copied().collect::<Vec<_>>()[..] else {
|
||
return None;
|
||
};
|
||
// Derived ONCE and used for both the identity lookup and the return: the two want the same value, and
|
||
// spelling it twice is how they came to disagree — the lookup was passed the FILE name against a map
|
||
// keyed by the short one, so it missed on every call and the guard below never rejected anything.
|
||
let lib = lib_name_from_file(&fname);
|
||
if let Some(id) = ident
|
||
&& id.contradiction(&lib, img, addr, &f.name).is_some()
|
||
{
|
||
return None;
|
||
}
|
||
Some((img, addr, lib))
|
||
}
|
||
|
||
/// Every distinct target address a unique-matching era-sig lands on, ranked by how many distinct
|
||
/// era-sigs voted for it (most first). Unlike `locate_addr` this keeps ALL candidates so a caller
|
||
/// can disambiguate a stale sig's coincidental hit from the real function by fingerprint.
|
||
fn locate_candidates<'a>(
|
||
prof: &GameProfile,
|
||
f: &Func,
|
||
images: &'a HashMap<String, CodeImage>,
|
||
) -> Option<(&'a CodeImage, String, VoteList)> {
|
||
let (img, fname) = candidate_libs(prof, f)
|
||
.into_iter()
|
||
.find_map(|fname| images.get(&fname).map(|img| (img, fname)))?;
|
||
let hits = scan_sig_hits(f, img);
|
||
if hits.is_empty() {
|
||
return None;
|
||
}
|
||
let mut cand: Vec<(u64, usize)> = hits.into_iter().collect();
|
||
cand.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0))); // most-voted first, addr as tiebreak
|
||
Some((img, lib_name_from_file(&fname), cand))
|
||
}
|
||
|
||
fn load_catalogue(path: &Path) -> Result<Vec<Func>> {
|
||
let text = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
|
||
let funcs: Vec<Func> = serde_json::from_str(&text).context("parse catalogue json")?;
|
||
// A name may appear ONCE. The catalogue is keyed by name everywhere downstream — `gd.entries`, the
|
||
// flag lists, the completeness sweep — but the distill walks this Vec, so a repeat pushes that name's
|
||
// slot observation twice and gives one function two votes in its own timeline. `gamedata` already
|
||
// merges same-named contributions into one `Func`'s variants for exactly this reason; a duplicate
|
||
// reaching here means the catalogue itself is malformed, which is an intake error rather than
|
||
// something to silently average out. Same posture as `is_iso_date` and the closed `ContributionKind`.
|
||
let mut seen: HashSet<&str> = HashSet::with_capacity(funcs.len());
|
||
let dups: Vec<&str> = funcs
|
||
.iter()
|
||
.map(|f| f.name.as_str())
|
||
.filter(|n| !seen.insert(n))
|
||
.collect();
|
||
ensure!(
|
||
dups.is_empty(),
|
||
"catalogue {} carries {} duplicate name(s): {}. Every consumer keys on the name, and the distill \
|
||
walks the list — a repeat gives one function two votes in its own slot timeline.",
|
||
path.display(),
|
||
dups.len(),
|
||
dups.join(", ")
|
||
);
|
||
Ok(funcs)
|
||
}
|
||
|
||
/// The two locator kinds a human may contribute. DELIBERATELY not `VariantKind`: that one carries a
|
||
/// `#[serde(other)]` catch-all, which is right for the machine-generated catalogue (forward-compatible with
|
||
/// kinds this build doesn't know) and WRONG for a hand-authored file — `"vtable_offset"` with an underscore,
|
||
/// or the real-but-underivable `"offset"`, would parse cleanly, be counted as merged, and then match nothing
|
||
/// downstream, producing no output and no diagnostic. A closed enum turns that typo into a parse error
|
||
/// naming the file and entry.
|
||
#[derive(Deserialize, Clone, Copy)]
|
||
#[serde(rename_all = "kebab-case")]
|
||
enum ContributionKind {
|
||
Signature,
|
||
VtableOffset,
|
||
}
|
||
|
||
impl From<ContributionKind> for VariantKind {
|
||
fn from(k: ContributionKind) -> Self {
|
||
match k {
|
||
ContributionKind::Signature => VariantKind::Signature,
|
||
ContributionKind::VtableOffset => VariantKind::VtableOffset,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// One human-contributed function locator, from `mappings/contributions/<game_key>/*.json`: a name + a
|
||
/// DATED signature or vtable-offset locator. Merged into the catalogue at DERIVE time — the unified
|
||
/// contribution/PR path. The `date` is the observation's build; the derive chains/verifies it exactly like
|
||
/// any dated catalogue variant.
|
||
#[derive(Deserialize)]
|
||
struct Contribution {
|
||
name: String,
|
||
kind: ContributionKind,
|
||
value: String,
|
||
date: String, // observation build date, YYYY-MM-DD (drives offset chaining / backfill)
|
||
}
|
||
|
||
/// `YYYY-MM-DD`, the shape the dated-variant machinery requires. Validated at INTAKE so a malformed date is
|
||
/// reported against its file and entry, rather than silently failing the `src.len() >= 10 && src[4] == b'-'`
|
||
/// test deep inside `vtable_offset_timelines`, where it just looks like a locator that didn't resolve.
|
||
fn is_iso_date(s: &str) -> bool {
|
||
let b = s.as_bytes();
|
||
b.len() == 10
|
||
&& b[4] == b'-'
|
||
&& b[7] == b'-'
|
||
&& b.iter()
|
||
.enumerate()
|
||
.all(|(i, c)| matches!(i, 4 | 7) || c.is_ascii_digit())
|
||
}
|
||
|
||
/// Human contributions for the active game as catalogue `Func`s (one dated variant each), read from
|
||
/// `<catalogue-dir>/contributions/<game_key>/*.json` in sorted order. Empty when the dir is absent.
|
||
/// Merged by the DERIVE only — never into the historical corpus model.
|
||
fn load_contributions(prof: &GameProfile, catalogue: &Path) -> Vec<Func> {
|
||
let Some(dir) = catalogue
|
||
.parent()
|
||
.map(|p| p.join("contributions").join(prof.game_key))
|
||
.filter(|d| d.is_dir())
|
||
else {
|
||
return Vec::new();
|
||
};
|
||
let mut files: Vec<PathBuf> = std::fs::read_dir(&dir)
|
||
.into_iter()
|
||
.flatten()
|
||
.flatten()
|
||
.map(|e| e.path())
|
||
.filter(|p| p.extension().is_some_and(|x| x == "json"))
|
||
.collect();
|
||
files.sort(); // deterministic merge order
|
||
let mut funcs = Vec::new();
|
||
for f in &files {
|
||
let text = match std::fs::read_to_string(f) {
|
||
Ok(t) => t,
|
||
Err(e) => {
|
||
eprintln!(" contribution {}: {e} — skipped", f.display());
|
||
continue;
|
||
}
|
||
};
|
||
// Parse entry-by-entry via `Vec<Value>` so ONE malformed entry costs only itself, not the whole
|
||
// file — CONTRIBUTING.md promises "a malformed entry is skipped with a warning".
|
||
let rows: Vec<Value> = match serde_json::from_str(&text) {
|
||
Ok(r) => r,
|
||
Err(e) => {
|
||
eprintln!(
|
||
" contribution {}: not a JSON array ({e}) — file skipped",
|
||
f.display()
|
||
);
|
||
continue;
|
||
}
|
||
};
|
||
for (i, row) in rows.into_iter().enumerate() {
|
||
let c: Contribution = match serde_json::from_value(row) {
|
||
Ok(c) => c,
|
||
Err(e) => {
|
||
eprintln!(" contribution {}[{i}]: {e} — entry skipped", f.display());
|
||
continue;
|
||
}
|
||
};
|
||
if !is_iso_date(&c.date) {
|
||
eprintln!(
|
||
" contribution {}[{i}] ({}): date {:?} is not YYYY-MM-DD — entry skipped",
|
||
f.display(),
|
||
c.name,
|
||
c.date
|
||
);
|
||
continue;
|
||
}
|
||
let variant = Variant {
|
||
kind: c.kind.into(),
|
||
platform: Platform::Linux,
|
||
value: c.value,
|
||
src: c.date,
|
||
};
|
||
// Merge onto an existing contributed Func rather than appending a duplicate. A second `Func`
|
||
// of the same name would never let the contribution corroborate the catalogue's own era-sigs
|
||
// (`scan_sig_hits` votes across the variants of ONE `Func`), and would let one name be counted
|
||
// twice by the emit tallies.
|
||
match funcs.iter_mut().find(|g: &&mut Func| g.name == c.name) {
|
||
Some(g) => g.variants.push(variant),
|
||
None => funcs.push(Func {
|
||
name: c.name,
|
||
library: Vec::new(),
|
||
variants: vec![variant],
|
||
}),
|
||
}
|
||
}
|
||
}
|
||
if !funcs.is_empty() {
|
||
eprintln!(
|
||
" merged {} contribution(s) from {}",
|
||
funcs.len(),
|
||
dir.display()
|
||
);
|
||
}
|
||
funcs
|
||
}
|
||
|
||
/// Immediate subdirectories of `corpus` that contain a libserver.so (i.e. builds), sorted.
|
||
pub(crate) fn find_builds(prof: &GameProfile, corpus: &Path) -> Result<Vec<PathBuf>> {
|
||
let mut builds: Vec<PathBuf> = std::fs::read_dir(corpus)
|
||
.with_context(|| format!("read {}", corpus.display()))?
|
||
.flatten()
|
||
.map(|e| e.path())
|
||
.filter(|p| p.is_dir() && find_file(p, prof.server_lib, 8).is_some())
|
||
.collect();
|
||
builds.sort();
|
||
ensure!(
|
||
!builds.is_empty(),
|
||
"no build subdirs containing libserver.so under {}",
|
||
corpus.display()
|
||
);
|
||
Ok(builds)
|
||
}
|
||
|
||
/// Load a build's server/engine libraries once (reused across many class lookups).
|
||
fn load_build_images(prof: &GameProfile, dir: &Path) -> Vec<CodeImage> {
|
||
load_build_images_counted(prof, dir).0
|
||
}
|
||
|
||
/// As [`load_build_images`], plus the libs that were absent or unparseable.
|
||
///
|
||
/// A build is admitted to the corpus on file EXISTENCE alone, so a corrupt-but-present library still
|
||
/// counts as a build; it then contributes empty fingerprints, which the distill folds in as slot-count 0
|
||
/// with an EMPTY hop for every class — and an empty hop is an unconditional chain break, severing every
|
||
/// anchor older than that build. That is worth a line of output rather than a silent `filter_map`.
|
||
fn load_build_images_counted(prof: &GameProfile, dir: &Path) -> (Vec<CodeImage>, Vec<String>) {
|
||
let mut imgs = Vec::new();
|
||
let mut failed = Vec::new();
|
||
for f in prof.libs {
|
||
// Absent is normal — a profile's lib list is a superset across builds — so only an
|
||
// unparseable file counts as a failure.
|
||
if let Some(p) = find_file(dir, f, 8) {
|
||
match CodeImage::load(&p) {
|
||
Ok(i) => imgs.push(i),
|
||
Err(_) => failed.push((*f).to_string()),
|
||
}
|
||
}
|
||
}
|
||
(imgs, failed)
|
||
}
|
||
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// §3 · FINGERPRINT ALIGNMENT
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
|
||
fn l1(a: &[u32], b: &[u32]) -> u64 {
|
||
a.iter().zip(b).map(|(&x, &y)| x.abs_diff(y) as u64).sum()
|
||
}
|
||
|
||
/// L1 distance, but stops as soon as it passes `cap` (then returns `cap + 1`). `slot_score` only needs
|
||
/// to know which of `{0, 1..=4, >4}` the distance lands in, and a Needleman–Wunsch fill scores mostly
|
||
/// unrelated slot pairs whose distance blows past the cap in the first few of 63 dims — so the early
|
||
/// exit turns the reduction from 63 adds into 1–3 on the hot path. Exact for the buckets `slot_score` uses.
|
||
fn l1_capped(a: &[u32], b: &[u32], cap: u64) -> u64 {
|
||
let mut sum = 0u64;
|
||
for (&x, &y) in a.iter().zip(b) {
|
||
sum += x.abs_diff(y) as u64;
|
||
if sum > cap {
|
||
return cap + 1;
|
||
}
|
||
}
|
||
sum
|
||
}
|
||
|
||
/// Score two slot fingerprints for alignment: identical → strong match, near → weak, else
|
||
/// mismatch. Undecodable slots never match.
|
||
fn slot_score(a: &Option<Vec<u32>>, b: &Option<Vec<u32>>) -> i32 {
|
||
match (a, b) {
|
||
(Some(x), Some(y)) => match l1_capped(x, y, 4) {
|
||
0 => 3,
|
||
1..=4 => 1,
|
||
_ => -1,
|
||
},
|
||
_ => -1,
|
||
}
|
||
}
|
||
|
||
/// Order-preserving (Needleman-Wunsch) alignment of two slot-fingerprint sequences, scored by
|
||
/// fingerprint similarity. Returns from_idx -> Some(to_idx) on diagonal (matched-position) steps,
|
||
/// None where the from-slot was deleted. Order preservation disambiguates the many near-identical
|
||
/// trivial getters a greedy nearest-neighbour cannot.
|
||
fn nw_align(from: &[Option<Vec<u32>>], to: &[Option<Vec<u32>>]) -> Vec<Option<usize>> {
|
||
let (n, m) = (from.len(), to.len());
|
||
// Identity fast-path: for equal sequences the all-diagonal path is the unique optimum (a match, incl.
|
||
// None-None, scores > a delete+insert gap pair at every prefix), so the traceback walks the full diagonal
|
||
// and returns `[Some(0)..Some(n-1)]`. This is BYTE-IDENTICAL to the full DP result — and on a code-distinct
|
||
// corpus the vast majority of a class's adjacent-build vtables are unchanged, so this skips the O(n²) fill.
|
||
if from == to {
|
||
return (0..n).map(Some).collect();
|
||
}
|
||
const GAP: i32 = -1;
|
||
// Flat (n+1)×(m+1) DP matrix in ONE allocation rather than n+1 heap Vecs — this is a per-call hot path
|
||
// (millions of calls per distill).
|
||
let w = m + 1;
|
||
let mut dp = vec![0i32; (n + 1) * w];
|
||
for i in 1..=n {
|
||
dp[i * w] = GAP * i as i32;
|
||
}
|
||
for j in 1..=m {
|
||
dp[j] = GAP * j as i32;
|
||
}
|
||
for i in 1..=n {
|
||
for j in 1..=m {
|
||
let s = slot_score(&from[i - 1], &to[j - 1]);
|
||
dp[i * w + j] = (dp[(i - 1) * w + (j - 1)] + s)
|
||
.max(dp[(i - 1) * w + j] + GAP)
|
||
.max(dp[i * w + (j - 1)] + GAP);
|
||
}
|
||
}
|
||
let mut map = vec![None; n];
|
||
let (mut i, mut j) = (n, m);
|
||
while i > 0 && j > 0 {
|
||
let s = slot_score(&from[i - 1], &to[j - 1]);
|
||
if dp[i * w + j] == dp[(i - 1) * w + (j - 1)] + s {
|
||
map[i - 1] = Some(j - 1); // diagonal: positions correspond (match or substitution)
|
||
i -= 1;
|
||
j -= 1;
|
||
} else if dp[i * w + j] == dp[(i - 1) * w + j] + GAP {
|
||
i -= 1;
|
||
} else {
|
||
j -= 1;
|
||
}
|
||
}
|
||
map
|
||
}
|
||
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// §4 · GAMEDATA FOLD & EXPERIMENTAL BAND (dead-weight/naming predicates live in `taxonomy`)
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
|
||
/// One verified promotable (T3) name from the name-extrapolation harvest.
|
||
#[derive(Deserialize)]
|
||
struct PromoName {
|
||
addr: String,
|
||
name: String,
|
||
#[serde(default)]
|
||
confidence: String,
|
||
#[serde(default)]
|
||
rationale: String,
|
||
#[serde(default)]
|
||
corroboration: String,
|
||
#[serde(default)]
|
||
self_named: bool,
|
||
#[serde(default)]
|
||
promote: bool,
|
||
}
|
||
|
||
/// One anonymous-function context bundle from the `--candidates` input (its membership = the dead-weight filter).
|
||
#[derive(Deserialize)]
|
||
struct CandCtx {
|
||
addr: String,
|
||
#[serde(default)]
|
||
class: Option<String>,
|
||
#[serde(default)]
|
||
slot: Option<i64>,
|
||
#[serde(default)]
|
||
abi: Option<String>,
|
||
}
|
||
|
||
#[derive(Deserialize, Default)]
|
||
struct CandDoc {
|
||
candidates: Vec<CandCtx>,
|
||
}
|
||
|
||
/// A function's ABI return class — typed so the by-value guard can't be silently broken by a `"byval"` string
|
||
/// typo. A struct returned BY VALUE hides an output-buffer pointer in RDI (RVO), so a naive `this`-call would
|
||
/// make the callee WRITE its return into the object = memory corruption; this flag keeps such functions out of
|
||
/// the blind-call path. Parsed from the candidate's `ret=` abi token; `as_str` round-trips to that token, so
|
||
/// the provenance `ret_class` string is byte-unchanged. Distinct from `abi::RetClass` (the DECODED 5-way
|
||
/// return class fed to the ABI-drift consensus): this one is parsed from the naming-harvest provenance token,
|
||
/// carries arbitrary type strings the fixed enum can't hold, and exists ONLY for the by-value safety flag.
|
||
enum ByValClass {
|
||
ByValue,
|
||
Other(String),
|
||
}
|
||
|
||
impl ByValClass {
|
||
fn from_abi(abi: Option<&str>) -> ByValClass {
|
||
match abi.and_then(parse_ret) {
|
||
Some("byval") => ByValClass::ByValue,
|
||
Some(s) => ByValClass::Other(s.to_string()),
|
||
None => ByValClass::Other("?".to_string()),
|
||
}
|
||
}
|
||
fn is_by_value(&self) -> bool {
|
||
matches!(self, ByValClass::ByValue)
|
||
}
|
||
fn as_str(&self) -> &str {
|
||
match self {
|
||
ByValClass::ByValue => "byval",
|
||
ByValClass::Other(s) => s,
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Does this promoted entry route to a vtable OFFSET (a clean-class method at a known slot whose AI-guessed
|
||
/// class matches its RTTI class) rather than a byte signature? Decided identically at the sig prefilter and
|
||
/// the emit loop.
|
||
fn is_offset_locator(prof: &GameProfile, pr: &PromoName, c: &CandCtx) -> bool {
|
||
let ai_class = pr.name.rsplit_once("::").map(|(cl, _)| cl);
|
||
c.slot.is_some()
|
||
&& c.class
|
||
.as_deref()
|
||
.is_some_and(|cls| clean_offset_class(prof, cls))
|
||
&& ai_class == c.class.as_deref()
|
||
}
|
||
|
||
/// Parse a `0x`-prefixed hex address string (the harvest/candidate `addr` field) to a `u64`. `None` on a
|
||
/// missing prefix or non-hex digits.
|
||
fn parse_hex_addr(s: &str) -> Option<u64> {
|
||
u64::from_str_radix(s.strip_prefix("0x")?, 16).ok()
|
||
}
|
||
|
||
/// A ground-truth vtable-OFFSET entry for a lib OTHER than the primary — `name` is `Class::Method` (the
|
||
/// class is read from the name, exactly like the primary fold's offset entries), `slot` is its vtable
|
||
/// index. Keyed by lib in the `--extra-offsets` file. The source is symbol transfer (macOS ground truth),
|
||
/// so these bypass the candidate/dead-weight gate — they are verified names, not guesses.
|
||
#[derive(Deserialize)]
|
||
struct ExtraOffset {
|
||
name: String,
|
||
class: String,
|
||
slot: i64,
|
||
/// Provenance overrides — absent = the macOS ground-truth shape (`corroborated`/`macos-groundtruth`),
|
||
/// so a plain `{name,class,slot}` stays macOS. AI-promotable virtual methods set their own tier
|
||
/// (`self-named`/`dict-exact`) + source so the monolith records where each name came from.
|
||
#[serde(default)]
|
||
tier: Option<String>,
|
||
#[serde(default)]
|
||
source: Option<String>,
|
||
#[serde(default)]
|
||
confidence: Option<String>,
|
||
#[serde(default)]
|
||
self_named: bool,
|
||
}
|
||
|
||
/// A multilib NON-virtual name to fold as a byte SIGNATURE — `addr` is its offset in lib `elib` (the
|
||
/// key). `make_sig` runs on that lib's own image. Used for AI-promotable names that aren't clean vtable
|
||
/// methods, keyed by lib in the `--extra-sigs` file.
|
||
#[derive(Deserialize)]
|
||
struct ExtraSig {
|
||
name: String,
|
||
addr: String,
|
||
#[serde(default)]
|
||
tier: Option<String>,
|
||
#[serde(default)]
|
||
source: Option<String>,
|
||
#[serde(default)]
|
||
confidence: Option<String>,
|
||
#[serde(default)]
|
||
self_named: bool,
|
||
}
|
||
|
||
/// Lift an extra-fold's data-string tier to the typed enum, erroring loudly on an unrecognised id.
|
||
fn tier_from_id(t: &str) -> Result<model::Tier> {
|
||
model::Tier::from_id(t).with_context(|| format!("unknown provenance tier {t:?}"))
|
||
}
|
||
|
||
/// Fold the multilib ground-truth vtable-OFFSET entries (macOS symbol transfer) for the non-primary server
|
||
/// libs as high_confidence offsets: the class is read from the name (so the modder resolves the right lib's
|
||
/// vtable) and the candidate/dead-weight gate is bypassed — these are verified ground truth, not guesses. A
|
||
/// name already owned by the promoted fold (`t3`) or the guaranteed core is left untouched. Returns the count.
|
||
fn fold_extra_offsets(
|
||
path: &Path,
|
||
core_gd: &GdMap,
|
||
t3: &mut BTreeMap<String, model::Entry>,
|
||
prov: &mut BTreeMap<String, model::Provenance>,
|
||
) -> Result<u32> {
|
||
let extra: BTreeMap<String, Vec<ExtraOffset>> =
|
||
serde_json::from_str(&std::fs::read_to_string(path)?)
|
||
.context("parse extra-offsets json")?;
|
||
let mut n = 0u32;
|
||
for (elib, entries) in &extra {
|
||
for e in entries {
|
||
if t3.contains_key(&e.name) || core_gd.contains_key(&e.name) {
|
||
continue; // a higher-precedence source already owns this name
|
||
}
|
||
t3.insert(e.name.clone(), model::Entry::offset(e.slot));
|
||
let source = e.source.as_deref().unwrap_or("macos-groundtruth");
|
||
prov.insert(
|
||
e.name.clone(),
|
||
model::Provenance {
|
||
addr: Some(String::new()),
|
||
confidence: Some(e.confidence.as_deref().unwrap_or("high").to_string()),
|
||
self_named: Some(e.self_named),
|
||
// NOT stated, because nothing here measured them. `by_value: false` is a SAFETY
|
||
// claim — "returns nothing by value, so a naive call is safe" — and this fold has
|
||
// only a name and a vtable slot, no address to measure at. The entry's `abi`, which
|
||
// `offset_abi_shapes` fills by resolving the slot, is the one statement about this
|
||
// function's return that rests on evidence.
|
||
by_value: None,
|
||
ret_class: None,
|
||
source: Some(source.to_string()),
|
||
rtti_class: Some(e.class.clone()),
|
||
rationale: Some(format!("multilib vtable offset ({elib}, {source})")),
|
||
..model::Provenance::with_tier(tier_from_id(
|
||
e.tier.as_deref().unwrap_or("corroborated"),
|
||
)?)
|
||
},
|
||
);
|
||
n += 1;
|
||
}
|
||
}
|
||
eprintln!(
|
||
" +{n} multilib ground-truth offset entries folded from {}",
|
||
path.display()
|
||
);
|
||
Ok(n)
|
||
}
|
||
|
||
/// Fold the multilib NON-virtual names as byte SIGNATURES via `make_sig` on each name's OWN lib image (loaded
|
||
/// on demand, cached) — the AI-promotable names that aren't clean vtable methods. Same precedence/gate rule as
|
||
/// [`fold_extra_offsets`]; a name whose sig can't be made (bad addr / no unique pattern) is skipped. Count.
|
||
fn fold_extra_sigs(
|
||
build: &Path,
|
||
path: &Path,
|
||
sig_cap: usize,
|
||
core_gd: &GdMap,
|
||
t3: &mut BTreeMap<String, model::Entry>,
|
||
prov: &mut BTreeMap<String, model::Provenance>,
|
||
) -> Result<u32> {
|
||
let extra: BTreeMap<String, Vec<ExtraSig>> =
|
||
serde_json::from_str(&std::fs::read_to_string(path)?).context("parse extra-sigs json")?;
|
||
let mut imgs: HashMap<String, CodeImage> = HashMap::new();
|
||
let mut n = 0u32;
|
||
// Losses here are the norm, not the exception (~30% of this input typically doesn't fold), and a
|
||
// success-only tally cannot distinguish "folded what it could" from "skipped an entire library".
|
||
let (mut skipped_libs, mut bad_addr, mut unmakeable, mut overlaps) =
|
||
(Vec::new(), 0u32, 0u32, 0u32);
|
||
for (elib, entries) in &extra {
|
||
if !imgs.contains_key(elib) {
|
||
match load_lib(build, elib) {
|
||
Ok(i) => {
|
||
imgs.insert(elib.clone(), i);
|
||
}
|
||
Err(_) => {
|
||
// lib not present in this build — its whole name list is skipped
|
||
skipped_libs.push(format!("{elib} ({})", entries.len()));
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
let eimg = &imgs[elib];
|
||
let elib_name = lib_name_from_file(elib);
|
||
for e in entries {
|
||
if t3.contains_key(&e.name) || core_gd.contains_key(&e.name) {
|
||
overlaps += 1;
|
||
continue;
|
||
}
|
||
let Some(addr) = parse_hex_addr(&e.addr) else {
|
||
bad_addr += 1;
|
||
continue;
|
||
};
|
||
let Some(sig) = emit::make_sig(eimg, addr, sig_cap) else {
|
||
unmakeable += 1;
|
||
continue;
|
||
};
|
||
let shape = abi::abi_shape(eimg, addr);
|
||
t3.insert(
|
||
e.name.clone(),
|
||
model::Entry::signature(elib_name.clone(), sig),
|
||
);
|
||
prov.insert(
|
||
e.name.clone(),
|
||
model::Provenance {
|
||
addr: Some(e.addr.clone()),
|
||
confidence: Some(e.confidence.as_deref().unwrap_or("medium").to_string()),
|
||
self_named: Some(e.self_named),
|
||
// MEASURED at the address this entry resolves to, not assumed. `by_value` says a
|
||
// naive call is safe; asserting `false` for every folded name would vouch for a
|
||
// by-value returner, which writes through a hidden pointer the caller never passed.
|
||
// Where the function does not decode, both stay `None` — an absent claim.
|
||
// THREE-way, not two: a decoded function whose return class is `Unknown` has not
|
||
// been shown safe either. `false` is a positive claim — "not a by-value returner, so
|
||
// a naive call will not write through a hidden pointer" — and it is only earned by a
|
||
// return class the measurement actually settled.
|
||
by_value: shape.and_then(|sh| match sh.ret_class {
|
||
abi::RetClass::Unknown => None,
|
||
c => Some(c == abi::RetClass::ByValue),
|
||
}),
|
||
ret_class: shape.map(|sh| sh.ret_class.describe().to_string()),
|
||
source: Some(
|
||
e.source
|
||
.as_deref()
|
||
.unwrap_or("source2rosetta-nameext")
|
||
.to_string(),
|
||
),
|
||
rationale: Some(format!("multilib name sig ({elib})")),
|
||
..model::Provenance::with_tier(tier_from_id(
|
||
e.tier.as_deref().unwrap_or("contextual"),
|
||
)?)
|
||
},
|
||
);
|
||
n += 1;
|
||
}
|
||
}
|
||
eprintln!(" +{n} multilib name sigs folded from {}", path.display());
|
||
if !skipped_libs.is_empty() || bad_addr > 0 || unmakeable > 0 || overlaps > 0 {
|
||
eprintln!(
|
||
" skipped: {overlaps} already-known, {unmakeable} unmakeable-sig, {bad_addr} bad-addr{}",
|
||
if skipped_libs.is_empty() {
|
||
String::new()
|
||
} else {
|
||
format!(", libs absent from this build: {}", skipped_libs.join(", "))
|
||
}
|
||
);
|
||
}
|
||
Ok(n)
|
||
}
|
||
|
||
// The provenance `source` ids. Declared HERE because this is where they are stamped, and `pub(crate)`
|
||
// because `prototypes` reads them back to decide what a name's evidence implies — it used to re-declare
|
||
// its own literals "kept in step" by a comment, which is an invariant nothing checks.
|
||
|
||
/// The provenance `source` id an entity-IO datadesc name ships under.
|
||
pub(crate) const VALVE_DATADESC: &str = "valve-datadesc";
|
||
|
||
/// The provenance `source` id prefix a console-command handler ships under; the callback FORM is
|
||
/// appended after a colon (`valve-concommand:direct`).
|
||
pub(crate) const VALVE_CONCOMMAND: &str = "valve-concommand";
|
||
|
||
/// Provenance id for a name the SCRIPT VM registry states. Same class of evidence as the datadesc and
|
||
/// console-command tables: the binary names the function and gives its address in one initialiser.
|
||
pub(crate) const VALVE_VSCRIPT: &str = "valve-vscript";
|
||
|
||
/// A console-command handler's gamedata name. The command name is Valve's own string; the prefix marks
|
||
/// what the entry IS — the handler bound to console command `X` — rather than claiming a C++ symbol we
|
||
/// do not know. `ent_fire`'s real method name is nowhere in the binary; `ConCommand::ent_fire` is.
|
||
fn concmd_key(name: &str) -> String {
|
||
format!("ConCommand::{name}")
|
||
}
|
||
|
||
/// Integer arguments in the entity-IO input prototype every datadesc handler shares —
|
||
/// `void(CEntityInstance*, InputData_t&)`: two pointers, no float arguments, nothing on the stack.
|
||
///
|
||
/// Hundreds of independent functions declared to one fixed prototype is a free correctness oracle for
|
||
/// the backward-liveness ABI reader, so the fold MEASURES them rather than assuming. What it checks is
|
||
/// the direction that matters: an OVER-count. The footprint is a documented lower bound — a handler
|
||
/// that ignores its `InputData_t&` genuinely reads one register, and a forwarding thunk reads none — so
|
||
/// measuring fewer arguments than declared is correct behaviour, while measuring MORE (or any float, or
|
||
/// a stack argument, or a by-value return) means the reader invented a parameter that isn't there, and
|
||
/// a consumer calling through it would load registers the function never reads.
|
||
///
|
||
/// Reported, not gated: a future build could legitimately introduce a handler this doesn't cover, and a
|
||
/// derivation should not fail on a diagnostic.
|
||
const IO_HANDLER_INT_ARGS: u8 = 2;
|
||
|
||
/// Does a measured shape stay within the entity-IO handler prototype (i.e. no over-count)?
|
||
fn within_io_prototype(s: &abi::AbiShape) -> bool {
|
||
s.int_args <= IO_HANDLER_INT_ARGS
|
||
&& s.float_args == 0
|
||
&& !s.stack_args
|
||
&& s.ret_class != abi::RetClass::ByValue
|
||
}
|
||
|
||
/// Integer arguments in a console-command callback: `void(const CCommandContext&, const CCommand&)`,
|
||
/// plus a receiver for the two forms that dispatch through an object.
|
||
///
|
||
/// The same free oracle the entity-IO prototype gives, from a second direction — and this one is
|
||
/// TWO-SIDED, which the entity-IO check is not. The form comes from the callback-type argument at the
|
||
/// registration site; the argument count comes from a backward-liveness pass over a completely
|
||
/// different function body, and neither reader knows the other exists. So a `direct` registration whose
|
||
/// handler reads three integer registers would mean one of the two is wrong — either the wrong function
|
||
/// was picked up, or the shape reader over-counted. Measured on CS2: **direct 699/699 within two,
|
||
/// interface and member 85/85 within three, zero float arguments and zero by-value returns anywhere.**
|
||
fn concmd_int_args(form: concmd::CallbackForm) -> u8 {
|
||
match form {
|
||
concmd::CallbackForm::Direct => 2,
|
||
concmd::CallbackForm::Interface | concmd::CallbackForm::Member => 3,
|
||
}
|
||
}
|
||
|
||
/// `PulseValueType_t::PVAL_EHANDLE`. Checkable rather than assumed: the enum is schema-registered, so
|
||
/// the `schema.enums` section of the shipped artifact states the same value.
|
||
const PVAL_EHANDLE: i32 = 13;
|
||
|
||
/// `mappings/ehandle-classes.json` — Valve's own naming for the entity class behind each handle
|
||
/// parameter, keyed by binding and parameter NAME. See `archive/valve-tables/extract-ehandle-classes.py`
|
||
/// for why it is keyed that way and not by address.
|
||
#[derive(serde::Deserialize)]
|
||
struct EhandleClasses {
|
||
classes: BTreeMap<String, BTreeMap<String, String>>,
|
||
}
|
||
|
||
/// Name the entity class behind every `PVAL_EHANDLE` parameter.
|
||
///
|
||
/// The initializer states a parameter's `PulseValueType_t` and, for a handle, stops there — the engine
|
||
/// resolves the entity class at runtime through a registry. But it also stores the address of the
|
||
/// TYPE'S DESTRUCTOR beside each parameter, and that address is per concrete type: two parameters carry
|
||
/// the same one exactly when they are the same handle type. So the binary supplies the GROUPING and
|
||
/// Valve's published metadata supplies the NAME, and neither alone is enough.
|
||
///
|
||
/// Grouped per LIBRARY because the token is an address: tokens from `libserver` and `libengine2` live
|
||
/// in different address spaces and comparing them would merge unrelated types.
|
||
///
|
||
/// Returns `(named, propagated, conflicts)`. A conflict is a group whose members Valve names
|
||
/// differently — which would mean the destructor grouping and the published naming disagree about what
|
||
/// is the same type, so NOTHING in that group is named. Measured 0 on both games across 44 and 41
|
||
/// groups, which is what makes this a standing two-sided oracle rather than a one-way lookup: the names
|
||
/// come from Valve and the grouping from the binary, and neither reader knows the other exists.
|
||
fn name_ehandle_classes(
|
||
bindings: &mut BTreeMap<String, model::Binding>,
|
||
table: &EhandleClasses,
|
||
) -> (u32, u32, Vec<String>) {
|
||
// (library, token) -> every handle parameter proven to be that type
|
||
let mut groups: BTreeMap<(String, u64), Vec<(String, bool, usize)>> = BTreeMap::new();
|
||
for (name, b) in bindings.iter() {
|
||
for (is_ret, list) in [(false, &b.params), (true, &b.returns)] {
|
||
for (i, p) in list.iter().enumerate() {
|
||
if p.ty == PVAL_EHANDLE && p.type_token != 0 {
|
||
groups
|
||
.entry((b.library.clone(), p.type_token))
|
||
.or_default()
|
||
.push((name.clone(), is_ret, i));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
let (mut named, mut propagated) = (0u32, 0u32);
|
||
let mut conflicts = Vec::new();
|
||
for ((lib, token), members) in &groups {
|
||
let stated: BTreeSet<&str> = members
|
||
.iter()
|
||
.filter_map(|(n, is_ret, i)| {
|
||
let p = &bindings[n];
|
||
let pname = &if *is_ret { &p.returns } else { &p.params }[*i].name;
|
||
table.classes.get(n)?.get(pname).map(String::as_str)
|
||
})
|
||
.collect();
|
||
match stated.len() {
|
||
0 => {}
|
||
1 => {
|
||
let cls = *stated.iter().next().expect("exactly one");
|
||
// `*` is Valve saying "any entity" — a statement, and one that must not be written out
|
||
// as though it were a class name.
|
||
if cls == "*" {
|
||
continue;
|
||
}
|
||
for (n, is_ret, i) in members {
|
||
let b = bindings.get_mut(n).expect("member came from this map");
|
||
let p = &mut if *is_ret {
|
||
&mut b.returns
|
||
} else {
|
||
&mut b.params
|
||
}[*i];
|
||
let known = table.classes.get(n).and_then(|m| m.get(&p.name)).is_some();
|
||
p.entity_class = Some(cls.to_string());
|
||
if known {
|
||
named += 1;
|
||
} else {
|
||
propagated += 1;
|
||
}
|
||
}
|
||
}
|
||
_ => conflicts.push(format!(
|
||
"{lib}+{token:#x}: Valve names this one type {:?}",
|
||
stated.iter().collect::<Vec<_>>()
|
||
)),
|
||
}
|
||
}
|
||
(named, propagated, conflicts)
|
||
}
|
||
|
||
/// Read Valve's OWN in-binary tables — the highest-ranking naming source the deriver has — folding the
|
||
/// names they LOCATE and shipping the surface they merely DOCUMENT.
|
||
///
|
||
/// Unlike every other fold input this reads no external file: the tables travel INSIDE each library, so
|
||
/// the stage re-runs on every build with nothing to maintain and nothing to bootstrap.
|
||
///
|
||
/// Only the entity-IO datadesc yields locators (see [`valvetab::names`] for why the Pulse table does
|
||
/// not); each of its names gets a fresh `make_sig` at the address Valve paired it with, plus a MEASURED
|
||
/// argument footprint rather than the placeholder a declared-only source can offer. The Pulse bindings
|
||
/// become the registry artifact instead.
|
||
///
|
||
/// Folding runs BEFORE the extrapolation fold so ground truth from THIS binary outranks inference — a
|
||
/// name the binary supplies itself beats the same name guessed by cross-game transfer or the naming pass.
|
||
fn fold_valve_tables(
|
||
prof: &GameProfile,
|
||
build: &Path,
|
||
source_build: &str,
|
||
sig_cap: usize,
|
||
t3: &mut BTreeMap<String, model::Entry>,
|
||
prov: &mut BTreeMap<String, model::Provenance>,
|
||
) -> ValveTables {
|
||
let mut out = ValveTables {
|
||
abi: BTreeMap::new(),
|
||
bindings: model::Bindings {
|
||
meta: model::BindingsMeta {
|
||
game_key: prof.game_key.to_string(),
|
||
source_build: source_build.to_string(),
|
||
pulse: 0,
|
||
pulse_typed: 0,
|
||
pulse_callable: 0,
|
||
entity_inputs: 0,
|
||
entity_outputs: 0,
|
||
entity_classes: 0,
|
||
commands: 0,
|
||
convars: 0,
|
||
vscript: 0,
|
||
vscript_located: 0,
|
||
vscript_classed: 0,
|
||
},
|
||
pulse: BTreeMap::new(),
|
||
entity_inputs: Vec::new(),
|
||
entity_outputs: Vec::new(),
|
||
entity_classes: BTreeMap::new(),
|
||
commands: Vec::new(),
|
||
convars: Vec::new(),
|
||
vscript: Vec::new(),
|
||
},
|
||
};
|
||
let (mut folded, mut ambiguous, mut unmakeable) = (0u32, 0usize, 0u32);
|
||
let (mut cmd_total, mut cmd_folded, mut cmd_dup, mut cmd_unmakeable) = (0u32, 0u32, 0u32, 0u32);
|
||
let mut vscript_folded = 0u32;
|
||
let (mut cmd_measured, mut cmd_agree) = (0u32, 0u32);
|
||
let (mut io_measured, mut io_agree) = (0u32, 0u32);
|
||
let (mut out_resolved, mut out_total) = (0u32, 0u32);
|
||
let (mut pulse_typed, mut pulse_total) = (0u32, 0u32);
|
||
let mut pulse_stride: Vec<(String, u64, usize, usize)> = Vec::new();
|
||
let (mut recv_agree, mut recv_total) = (0u32, 0u32);
|
||
let (mut dd_qualified, mut dd_total) = (0u32, 0u32);
|
||
let mut dup_regs = 0u32;
|
||
let mut dup_conflicts: Vec<String> = Vec::new();
|
||
let mut dup_names: BTreeSet<String> = BTreeSet::new();
|
||
let mut unreadable: Vec<String> = Vec::new();
|
||
let threads = default_threads(None);
|
||
|
||
for f in prof.libs {
|
||
// A profile's lib list is a superset across builds and games, so an ABSENT library is normal and
|
||
// silent. A library that is present and does not PARSE is not: it would drop that module's whole
|
||
// declared surface — its commands, its Pulse bindings, its datadesc — and the artifact would look
|
||
// exactly like a build where the module registered nothing. Say which one it was.
|
||
let img = match find_file(build, f, 8) {
|
||
None => continue,
|
||
Some(path) => match CodeImage::load(&path) {
|
||
Ok(img) => img,
|
||
Err(e) => {
|
||
unreadable.push(format!("{f} ({e:#})"));
|
||
continue;
|
||
}
|
||
},
|
||
};
|
||
// Console commands come first because they are the one source here that is independent of the
|
||
// static tables: a library can register a hundred commands and hold neither a Pulse binding nor
|
||
// a datadesc array (libsoundsystem does exactly that), so this must not sit behind the
|
||
// table-emptiness skip below.
|
||
let commands = concmd::console_commands(&img);
|
||
let lib = lib_name_from_file(f);
|
||
// ConVars: the other half of the console surface, read by the same pass. Documentation, not a
|
||
// locator — a consumer finds a convar by name at runtime; the flags are what it cannot get itself.
|
||
out.bindings
|
||
.convars
|
||
.extend(
|
||
concmd::convars(&img, &lib)
|
||
.into_iter()
|
||
.map(|c| model::ConVar {
|
||
name: c.name,
|
||
library: c.library,
|
||
description: c.description,
|
||
flags: c.flags,
|
||
flags_raw: c.flags_raw,
|
||
addr: c.addr,
|
||
}),
|
||
);
|
||
{
|
||
let addrs: Vec<u64> = commands.iter().map(|c| c.handler).collect();
|
||
let sigs = parallel_map(&addrs, threads, |&a| emit::make_sig(&img, a, sig_cap));
|
||
let shapes = parallel_map(&addrs, threads, |&a| abi::abi_shape(&img, a));
|
||
for ((c, sig), shape) in commands.iter().zip(sigs).zip(shapes) {
|
||
cmd_total += 1;
|
||
// The oracle runs over EVERY command, including the ones that do not fold — it is about
|
||
// the measurement, not the name.
|
||
if let Some(s) = shape {
|
||
cmd_measured += 1;
|
||
cmd_agree += u32::from(
|
||
s.int_args <= concmd_int_args(c.form)
|
||
&& s.float_args == 0
|
||
&& !s.stack_args
|
||
&& s.ret_class != abi::RetClass::ByValue,
|
||
);
|
||
}
|
||
let key = concmd_key(&c.name);
|
||
out.bindings.commands.push(model::ConsoleCommand {
|
||
name: c.name.clone(),
|
||
library: lib.clone(),
|
||
description: c.description.clone(),
|
||
flags: concmd::flag_names(c.flags)
|
||
.into_iter()
|
||
.map(str::to_string)
|
||
.collect(),
|
||
flags_raw: format!("{:#x}", c.flags),
|
||
form: c.form.describe().to_string(),
|
||
abi: shape.map(shipped_abi),
|
||
addr: format!("{:#x}", c.handler),
|
||
});
|
||
if t3.contains_key(&key) {
|
||
// A name registered by several libraries keeps the first, in the profile's
|
||
// resolution order — the same rule the datadesc names follow.
|
||
cmd_dup += 1;
|
||
continue;
|
||
}
|
||
let Some(sig) = sig else {
|
||
cmd_unmakeable += 1;
|
||
continue;
|
||
};
|
||
t3.insert(key.clone(), model::Entry::signature(lib.clone(), sig));
|
||
prov.insert(
|
||
key,
|
||
model::Provenance {
|
||
addr: Some(format!("{:#x}", c.handler)),
|
||
confidence: Some("high".to_string()),
|
||
// The binary states the name and the address in the same instruction sequence.
|
||
self_named: Some(true),
|
||
// The FORM rides in the source id (the `contribution:<date>` shape), because
|
||
// the prototype manifest needs it: a direct callback takes two arguments and
|
||
// the two object forms take a receiver as well. See
|
||
// `prototypes::concommand_contract`.
|
||
source: Some(format!("{VALVE_CONCOMMAND}:{}", c.form.describe())),
|
||
rationale: Some(format!(
|
||
"registered as console command {:?} in {f} ({} callback)",
|
||
c.name,
|
||
c.form.describe()
|
||
)),
|
||
..model::Provenance::with_tier(model::Tier::ValveTable)
|
||
},
|
||
);
|
||
if let Some(s) = shape {
|
||
out.abi.insert(concmd_key(&c.name), shipped_abi(s));
|
||
}
|
||
cmd_folded += 1;
|
||
}
|
||
}
|
||
|
||
// VScript — the script VM's registry. Folded exactly like the console commands: the binary
|
||
// states the name and the implementation in one initialiser, so the entry is `valve-table`
|
||
// provenance and self-named.
|
||
//
|
||
// Keyed by the C++ name, not the script-facing one. The script name is what a Lua author types
|
||
// and belongs in the binding row; the C++ name is what actually lives at the address, and
|
||
// gamedata names functions. The two never collide with existing entries — measured at ZERO
|
||
// shared addresses and ZERO shared names against the catalogue, because a
|
||
// `Script_TakeDamage` is a wrapper and not the `TakeDamage` it wraps.
|
||
{
|
||
let vsf = crate::vscript::vscript_functions(&img);
|
||
// A C++ name registered at more than one address cannot be keyed honestly, so it is dropped
|
||
// rather than resolved by fiat — the rule the ambiguous datadesc handlers already follow. On
|
||
// the current builds this is 2 of 1,650 on Dota and 0 of 256 on CS2.
|
||
let mut addrs_of: HashMap<&str, HashSet<u64>> = HashMap::new();
|
||
for f in &vsf {
|
||
if let Some(crate::vscript::Impl::Addr(a)) = f.imp {
|
||
addrs_of.entry(f.cpp_name.as_str()).or_default().insert(a);
|
||
}
|
||
}
|
||
let foldable: Vec<&crate::vscript::VScriptFunc> = vsf
|
||
.iter()
|
||
.filter(|f| {
|
||
matches!(f.imp, Some(crate::vscript::Impl::Addr(_)))
|
||
&& addrs_of
|
||
.get(f.cpp_name.as_str())
|
||
.is_some_and(|s| s.len() == 1)
|
||
})
|
||
.collect();
|
||
let vaddrs: Vec<u64> = foldable
|
||
.iter()
|
||
.filter_map(|f| match f.imp {
|
||
Some(crate::vscript::Impl::Addr(a)) => Some(a),
|
||
_ => None,
|
||
})
|
||
.collect();
|
||
let vsigs = parallel_map(&vaddrs, threads, |&a| emit::make_sig(&img, a, sig_cap));
|
||
for (f, sig) in foldable.iter().zip(vsigs) {
|
||
let Some(sig) = sig else { continue };
|
||
if t3.contains_key(&f.cpp_name) {
|
||
continue;
|
||
}
|
||
let Some(crate::vscript::Impl::Addr(addr)) = f.imp else {
|
||
continue;
|
||
};
|
||
t3.insert(
|
||
f.cpp_name.clone(),
|
||
model::Entry::signature(lib.clone(), sig),
|
||
);
|
||
prov.insert(
|
||
f.cpp_name.clone(),
|
||
model::Provenance {
|
||
addr: Some(format!("{addr:#x}")),
|
||
confidence: Some("high".to_string()),
|
||
self_named: Some(true),
|
||
source: Some(VALVE_VSCRIPT.to_string()),
|
||
rationale: Some(format!(
|
||
"registered with the script VM as {:?} in {f_lib}",
|
||
f.name,
|
||
f_lib = lib
|
||
)),
|
||
..model::Provenance::with_tier(model::Tier::ValveTable)
|
||
},
|
||
);
|
||
vscript_folded += 1;
|
||
}
|
||
for f in &vsf {
|
||
out.bindings.vscript.push(model::VScriptBinding {
|
||
name: f.name.clone(),
|
||
// Filled in by the live oracle; not derivable here (see `VScriptBinding::class`).
|
||
class: None,
|
||
cpp: f.cpp_name.clone(),
|
||
library: lib.clone(),
|
||
description: f.description.clone().unwrap_or_default(),
|
||
ret: f.ret.map(str::to_string),
|
||
ret_raw: f.ret_raw,
|
||
addr: match f.imp {
|
||
Some(crate::vscript::Impl::Addr(a)) => Some(format!("{a:#x}")),
|
||
_ => None,
|
||
},
|
||
vtable_slot: match f.imp {
|
||
Some(crate::vscript::Impl::Slot(s)) => Some(s),
|
||
_ => None,
|
||
},
|
||
// Joined at view time, off the function record this row folds onto — see
|
||
// `VScriptBinding::doc`.
|
||
doc: None,
|
||
});
|
||
}
|
||
}
|
||
|
||
let pulse = valvetab::pulse_bindings(&img);
|
||
// The datadesc by ARRAY, so each handler can be attributed to the class that owns it. The
|
||
// record carries no owning class; the array's FIELD descriptors do, because a `(member, offset)`
|
||
// pair is something the SchemaSystem states from an entirely different table. Only an
|
||
// unambiguous fingerprint counts — an array several classes could satisfy names none of them.
|
||
let arrays = valvetab::datadesc_arrays(&img);
|
||
// ONE schema enumeration per library, shared by the array join below and the entity-OUTPUT join
|
||
// further down. It is a full reflection-table walk over a 133 MB image on Dota, so doing it
|
||
// twice per library across 21 libraries is not a slow path, it is an out-of-memory kill — which
|
||
// is exactly what an unguarded second call produced.
|
||
let sch = schema::enumerate_schema(&img);
|
||
let mut inputs: Vec<valvetab::DatadescInput> = Vec::new();
|
||
for arr in arrays {
|
||
let owners: Vec<&schema::SchemaClass> = if arr.fields.is_empty() {
|
||
Vec::new()
|
||
} else {
|
||
sch.iter()
|
||
.filter(|c| {
|
||
arr.fields
|
||
.iter()
|
||
.all(|(n, o)| c.fields.iter().any(|f| f.name == *n && f.offset == *o))
|
||
})
|
||
.collect()
|
||
};
|
||
let class = (owners.len() == 1).then(|| owners[0].name.clone());
|
||
dd_total += arr.inputs.len() as u32;
|
||
dd_qualified += u32::from(class.is_some()) * arr.inputs.len() as u32;
|
||
for mut i in arr.inputs {
|
||
i.class = class.clone();
|
||
inputs.push(i);
|
||
}
|
||
}
|
||
// An output's locator is a member OFFSET, which is meaningless without the class it sits on —
|
||
// and the SchemaSystem in this same image already states every member's class, offset and type.
|
||
// Joining the two is the composition that makes the offset usable, and it doubles as a free
|
||
// per-build oracle: the datadesc and the schema are independent reflection systems, so their
|
||
// agreement on a shared fact is evidence both readers are still correct.
|
||
let outputs = valvetab::datadesc_outputs(&img);
|
||
if !outputs.is_empty() {
|
||
let mut by_member: HashMap<(&str, i32), Vec<&schema::SchemaClass>> = HashMap::new();
|
||
for c in &sch {
|
||
for fl in &c.fields {
|
||
by_member
|
||
.entry((fl.name.as_str(), fl.offset))
|
||
.or_default()
|
||
.push(c);
|
||
}
|
||
}
|
||
for o in outputs {
|
||
// Only an UNAMBIGUOUS join is used: several classes declaring the same member at the same
|
||
// offset would put us back to guessing, which is the thing this exists to stop.
|
||
let hit = by_member
|
||
.get(&(o.member.as_str(), o.offset as i32))
|
||
.filter(|v| v.len() == 1)
|
||
.map(|v| v[0]);
|
||
out_resolved += u32::from(hit.is_some());
|
||
out_total += 1;
|
||
out.bindings.entity_outputs.push(model::EntityOutput {
|
||
output: o.output,
|
||
member: o.member.clone(),
|
||
library: lib_name_from_file(f),
|
||
offset: o.offset,
|
||
class: hit.map(|c| c.name.clone()),
|
||
});
|
||
}
|
||
}
|
||
for e in valvetab::entity_classes(&img) {
|
||
out.bindings
|
||
.entity_classes
|
||
.entry(e.classname)
|
||
.or_insert(e.class);
|
||
}
|
||
if pulse.is_empty() && inputs.is_empty() {
|
||
continue;
|
||
}
|
||
let (names, dropped) = valvetab::names(&inputs);
|
||
ambiguous += dropped;
|
||
|
||
// One measurement per distinct handler: an address recurs across records (the same handler
|
||
// serves several inputs). Sorted so the parallel pass is deterministic.
|
||
let mut addrs: Vec<u64> = inputs
|
||
.iter()
|
||
.map(|i| i.func)
|
||
.collect::<HashSet<u64>>()
|
||
.into_iter()
|
||
.collect();
|
||
addrs.sort_unstable();
|
||
let shape_at: HashMap<u64, abi::AbiShape> =
|
||
parallel_map(&addrs, threads, |&a| (a, abi::abi_shape(&img, a)))
|
||
.into_iter()
|
||
.filter_map(|(a, s)| s.map(|s| (a, s)))
|
||
.collect();
|
||
// make_sig dominates the cost and is a pure read on the image, so it runs across threads too;
|
||
// the result order follows the input, so the fold stays deterministic.
|
||
let name_addrs: Vec<u64> = names.iter().map(|n| n.addr).collect();
|
||
let sigs = parallel_map(&name_addrs, threads, |&a| emit::make_sig(&img, a, sig_cap));
|
||
|
||
// The oracle runs over EVERY handler, not just the ones that fold: an ambiguous name is still a
|
||
// function with a known prototype, and the check is about the measurement, not the name.
|
||
for i in &inputs {
|
||
if let Some(s) = shape_at.get(&i.func) {
|
||
io_measured += 1;
|
||
io_agree += u32::from(within_io_prototype(s));
|
||
}
|
||
}
|
||
|
||
for (n, sig) in names.iter().zip(sigs) {
|
||
let shape = shape_at.get(&n.addr).copied();
|
||
if t3.contains_key(&n.name) {
|
||
continue; // an earlier library's table already vouched for this name
|
||
}
|
||
let Some(sig) = sig else {
|
||
unmakeable += 1;
|
||
continue;
|
||
};
|
||
t3.insert(n.name.clone(), model::Entry::signature(lib.clone(), sig));
|
||
prov.insert(
|
||
n.name.clone(),
|
||
model::Provenance {
|
||
addr: Some(format!("{:#x}", n.addr)),
|
||
confidence: Some("high".to_string()),
|
||
// The binary names the function itself — a stronger claim than the naming pass's
|
||
// "the name appears in the function's own bytes", not a weaker one.
|
||
self_named: Some(true),
|
||
// DECLARED, not measured. The engine invokes every entity-IO handler through
|
||
// `void (CBaseEntity::*)(inputdata_t&)`, so the return value is never read — the
|
||
// prototype settles it. That is worth preferring here because the measured class is
|
||
// specifically weak on exactly this question: a `void` function using RAX as scratch
|
||
// reads back as `ret=int`, and across these same handlers only ~12% classify as void.
|
||
by_value: Some(false),
|
||
ret_class: Some(abi::RetClass::Void.describe().to_string()),
|
||
source: Some(VALVE_DATADESC.to_string()),
|
||
rationale: Some(format!("Valve entity-IO datadesc in {f}")),
|
||
..model::Provenance::with_tier(model::Tier::ValveTable)
|
||
},
|
||
);
|
||
if let Some(s) = shape {
|
||
out.abi.insert(n.name.clone(), shipped_abi(s));
|
||
}
|
||
folded += 1;
|
||
}
|
||
|
||
// The registry describes the callable SURFACE, so it ships every record — including the Pulse
|
||
// bindings that have no locator and the handlers whose bare name was too ambiguous to fold. An
|
||
// `InputEnable` on 48 classes is 48 real handlers; only naming them individually is impossible.
|
||
//
|
||
// Each binding's two accessors are read for its TYPED SIGNATURE, which is the one thing the
|
||
// record itself does not carry. Read together rather than one at a time, because the element
|
||
// stride is derived by consensus across the image — see `pulse::read_all`.
|
||
let pairs: Vec<(u64, u64)> = pulse
|
||
.iter()
|
||
.map(|b| (b.descriptor, b.arg_descriptor))
|
||
.collect();
|
||
let (sig_of, stride, stride_votes, stride_rivals) = pulse::read_all(&img, &pairs, threads);
|
||
if stride_votes > 0 {
|
||
pulse_stride.push((lib.clone(), stride, stride_votes, stride_rivals));
|
||
}
|
||
for (b, sig) in pulse.into_iter().zip(sig_of) {
|
||
pulse_typed += u32::from(sig.is_some());
|
||
pulse_total += 1;
|
||
let kind = binding_kind(b.flags);
|
||
// A FREE ORACLE, and a two-sided one. The receiver flag was decoded last session from naming
|
||
// conventions alone; the parameter list is read from a completely different place in the
|
||
// binary. If the flag means what it was inferred to mean, an `instance` binding takes a
|
||
// `_Target` receiver and nothing else does — so agreement is independent evidence for the
|
||
// flag decoding, and disagreement would mean the signature reader had shifted a parameter
|
||
// list. Measured 999/999 and 859/859 registrations, no exceptions in either game.
|
||
if let Some(s) = sig.as_ref() {
|
||
let has_target = s.args.first().is_some_and(|p| p.name == "_Target");
|
||
recv_total += 1;
|
||
recv_agree += u32::from(has_target == (kind == model::BindingKind::Instance));
|
||
}
|
||
let entry = model::Binding {
|
||
library: lib.clone(),
|
||
display: b.display,
|
||
description: b.description,
|
||
policy: model::CallPolicy {
|
||
kind,
|
||
mutates: b.flags.mutating,
|
||
blocking: b.flags.blocking,
|
||
raw: [b.flags.raw.0, b.flags.raw.1],
|
||
},
|
||
params: sig.as_ref().map(|s| s.args.clone()).unwrap_or_default(),
|
||
returns: sig.as_ref().map(|s| s.returns.clone()).unwrap_or_default(),
|
||
typed: sig.is_some(),
|
||
descriptor: format!("{:#x}", b.descriptor),
|
||
shim: (b.shim != 0).then(|| format!("{:#x}", b.shim)),
|
||
// Measured per shim rather than assumed from the tier: the calling contract is fixed, but
|
||
// WHICH slots a given shim reads is the whole difference between host-callable and not.
|
||
call: (b.shim != 0)
|
||
.then(|| pulse::shim_reads(&img, b.shim))
|
||
.flatten()
|
||
.map(|r| model::ShimCall {
|
||
needs: r.needs().to_string(),
|
||
reads: r.reads.iter().map(|s| s.to_string()).collect(),
|
||
}),
|
||
};
|
||
// The registry is keyed by qualified name across libraries, so a binding registered by
|
||
// more than one module keeps ONE row. That is the documented lossiness — of the LIBRARY,
|
||
// not of the signature — and it is checked rather than assumed: every duplicate is compared
|
||
// against the row already present and a disagreement is reported.
|
||
//
|
||
// IT IS REPORTING, on every derive of both games — CS2 flags 331 of 419 repeat
|
||
// registrations and Dota 271 of 359. So the artifact IS choosing between two accounts of
|
||
// one binding for those names, which is exactly the outcome this check exists to make
|
||
// impossible to miss. An earlier "0 disagreements" reading here predates the typed-signature
|
||
// recovery and was never revisited. Until it is, a consumer reading `params`/`returns` for a
|
||
// multiply-registered binding is reading ONE module's account of it, not a merged one.
|
||
// The FIRST library wins, which is the rule every sibling table in this loop already
|
||
// follows (commands, VScript, datadesc, entity classes, the live schema) and the one
|
||
// `GameProfile::libs` documents — "an earlier lib wins". Pulse was the lone `insert`, so the
|
||
// LAST registration overwrote everything, and 224 of 580 CS2 rows shipped another module's
|
||
// account under a name libserver also registers. It also broke the oracle pairing: the live
|
||
// shim and descriptor checks run against the SERVER image, so for exactly the duplicated
|
||
// names the rows that got verified were not the rows that got shipped.
|
||
//
|
||
// One exception, because precedence must not cost information: a typed row is strictly more
|
||
// than an untyped one, so a later library that DID recover a signature replaces an earlier
|
||
// one that did not. Never the reverse.
|
||
match out.bindings.pulse.get(&b.name) {
|
||
Some(prev) => {
|
||
dup_regs += 1;
|
||
dup_names.insert(b.name.clone());
|
||
if prev.params != entry.params
|
||
|| prev.returns != entry.returns
|
||
|| prev.typed != entry.typed
|
||
{
|
||
dup_conflicts.push(b.name.clone());
|
||
}
|
||
if entry.typed && !prev.typed {
|
||
out.bindings.pulse.insert(b.name, entry);
|
||
}
|
||
}
|
||
None => {
|
||
out.bindings.pulse.insert(b.name, entry);
|
||
}
|
||
}
|
||
}
|
||
for i in inputs {
|
||
out.bindings.entity_inputs.push(model::EntityInput {
|
||
input: i.io_name,
|
||
class: i.class,
|
||
handler: i.handler,
|
||
library: lib.clone(),
|
||
abi: shape_at.get(&i.func).copied().map(shipped_abi),
|
||
addr: format!("{:#x}", i.func),
|
||
});
|
||
}
|
||
}
|
||
out.bindings
|
||
.entity_inputs
|
||
.sort_by(|a, b| (&a.input, &a.handler, &a.addr).cmp(&(&b.input, &b.handler, &b.addr)));
|
||
out.bindings.meta.pulse = out.bindings.pulse.len();
|
||
out.bindings.meta.pulse_typed = out.bindings.pulse.values().filter(|b| b.typed).count();
|
||
// Counted from the emitted rows rather than tallied during the fold, so the number in `meta` cannot
|
||
// drift from the number of rows a consumer can actually act on.
|
||
out.bindings.meta.pulse_callable = out
|
||
.bindings
|
||
.pulse
|
||
.values()
|
||
.filter(|b| b.call.as_ref().is_some_and(|c| c.needs == "args-only"))
|
||
.count();
|
||
out.bindings.meta.entity_inputs = out.bindings.entity_inputs.len();
|
||
out.bindings
|
||
.entity_outputs
|
||
.sort_by(|a, b| (&a.output, &a.member).cmp(&(&b.output, &b.member)));
|
||
out.bindings.meta.entity_outputs = out.bindings.entity_outputs.len();
|
||
out.bindings.meta.entity_classes = out.bindings.entity_classes.len();
|
||
out.bindings
|
||
.commands
|
||
.sort_by(|a, b| (&a.name, &a.addr).cmp(&(&b.name, &b.addr)));
|
||
out.bindings.meta.commands = out.bindings.commands.len();
|
||
out.bindings.meta.convars = out.bindings.convars.len();
|
||
out.bindings.meta.vscript = out.bindings.vscript.len();
|
||
out.bindings.meta.vscript_located = vscript_folded as usize;
|
||
|
||
if cmd_total > 0 {
|
||
let libs: BTreeSet<&str> = out
|
||
.bindings
|
||
.commands
|
||
.iter()
|
||
.map(|c| c.library.as_str())
|
||
.collect();
|
||
eprintln!(
|
||
" +{cmd_folded} names from console-command registration ({cmd_total} commands across \
|
||
{} libraries)",
|
||
libs.len()
|
||
);
|
||
if cmd_dup > 0 || cmd_unmakeable > 0 {
|
||
eprintln!(
|
||
" skipped: {cmd_dup} already named by an earlier library, {cmd_unmakeable} \
|
||
unmakeable-sig"
|
||
);
|
||
}
|
||
eprintln!(
|
||
" +{vscript_folded} names from the script-VM registry ({} bindings, {} with a \
|
||
script-facing name Valve documents)",
|
||
out.bindings.vscript.len(),
|
||
out.bindings
|
||
.vscript
|
||
.iter()
|
||
.filter(|v| !v.description.is_empty())
|
||
.count()
|
||
);
|
||
// A STANDING ORACLE, and an intrinsic one — it needs no external list. Every command has its
|
||
// own handler, so commands and distinct handler addresses should track each other. The failure
|
||
// this catches is specific: the member-callback form finds its handler by scanning the accessor
|
||
// object, and a layout change there would start returning the shared dispatch thunk instead of
|
||
// the per-command function. That would still resolve, still validate live, and still be wrong —
|
||
// and it would show up here as a collapse in the second number while the first held steady.
|
||
let distinct: BTreeSet<&str> = out
|
||
.bindings
|
||
.commands
|
||
.iter()
|
||
.map(|c| c.addr.as_str())
|
||
.collect();
|
||
let by_form = |w: &str| out.bindings.commands.iter().filter(|c| c.form == w).count();
|
||
eprintln!(
|
||
" console-command handler distinctness: {} commands -> {} distinct handlers ({} \
|
||
direct, {} interface, {} member)",
|
||
cmd_total,
|
||
distinct.len(),
|
||
by_form("direct"),
|
||
by_form("interface"),
|
||
by_form("member"),
|
||
);
|
||
if cmd_measured > 0 {
|
||
eprintln!(
|
||
" console-command ABI oracle: {cmd_agree}/{cmd_measured} handlers measure within \
|
||
the callback prototype their registration declares ({:.1}%)",
|
||
100.0 * f64::from(cmd_agree) / f64::from(cmd_measured)
|
||
);
|
||
}
|
||
}
|
||
eprintln!(" +{folded} names from Valve's entity-IO datadesc");
|
||
if ambiguous > 0 || unmakeable > 0 {
|
||
eprintln!(
|
||
" skipped: {ambiguous} ambiguous (one name, several addresses — no owning class to \
|
||
disambiguate), {unmakeable} unmakeable-sig"
|
||
);
|
||
}
|
||
if io_measured > 0 {
|
||
eprintln!(
|
||
" entity-IO ABI oracle: {io_agree}/{io_measured} handlers measure within the declared \
|
||
void(2 int) footprint ({:.1}%)",
|
||
100.0 * f64::from(io_agree) / f64::from(io_measured)
|
||
);
|
||
}
|
||
if out_total > 0 {
|
||
eprintln!(
|
||
" entity-output schema join: {out_resolved}/{out_total} outputs resolved to exactly one \
|
||
class ({:.1}%) — two independent reflection systems agreeing on the same member",
|
||
100.0 * f64::from(out_resolved) / f64::from(out_total)
|
||
);
|
||
}
|
||
// OUTSIDE the Pulse block below, and that is exactly the point: a library that failed to parse
|
||
// contributes no Pulse registrations, so `pulse_total > 0` is the condition LEAST likely to hold when
|
||
// this warning is most needed. Nested there, it silenced itself in its own trigger case.
|
||
if !unreadable.is_empty() {
|
||
eprintln!(
|
||
" WARNING {} declared-surface librar{} PRESENT but unreadable, so their commands, Pulse \
|
||
bindings and datadesc are missing from this artifact rather than absent from the build: {}",
|
||
unreadable.len(),
|
||
if unreadable.len() == 1 {
|
||
"y is"
|
||
} else {
|
||
"ies are"
|
||
},
|
||
unreadable.join(", ")
|
||
);
|
||
}
|
||
if pulse_total > 0 {
|
||
// The stride is DERIVED per image, so report it: it is the one layout fact the reader takes from
|
||
// consensus rather than from each record, and a build that changed the record size would show up
|
||
// here as a new number before it showed up as lost signatures.
|
||
let strides: BTreeSet<u64> = pulse_stride.iter().map(|(_, s, _, _)| *s).collect();
|
||
let votes: usize = pulse_stride.iter().map(|(_, _, v, _)| v).sum();
|
||
let rivals: usize = pulse_stride.iter().map(|(_, _, _, r)| r).sum();
|
||
// REGISTRATIONS, not distinct bindings: a binding registered by several libraries is read once
|
||
// per library, so this population is larger than `meta.pulse`. Saying "bindings" here would put
|
||
// two different numbers under one word.
|
||
eprintln!(
|
||
" Pulse typed signatures: {pulse_typed}/{pulse_total} registrations recovered from their \
|
||
own descriptor initializers ({:.1}%); element stride {} agreed by {votes} records across \
|
||
{} libs, {rivals} dissenting",
|
||
100.0 * f64::from(pulse_typed) / f64::from(pulse_total),
|
||
strides
|
||
.iter()
|
||
.map(u64::to_string)
|
||
.collect::<Vec<_>>()
|
||
.join("/"),
|
||
pulse_stride.len()
|
||
);
|
||
eprintln!(
|
||
" Pulse receiver cross-check: {recv_agree}/{recv_total} registrations agree on whether \
|
||
they take a receiver ({:.1}%) — the `+56` flag byte and the recovered parameter list are \
|
||
read from different places and neither knows about the other",
|
||
100.0 * f64::from(recv_agree) / f64::from(recv_total.max(1))
|
||
);
|
||
if dup_conflicts.is_empty() {
|
||
eprintln!(
|
||
" Pulse multi-library duplicates: {dup_regs} repeat registrations across {} \
|
||
bindings, all agreeing with the row already read",
|
||
dup_names.len()
|
||
);
|
||
} else {
|
||
eprintln!(
|
||
" FLAG {} of {dup_regs} repeat registrations DISAGREE on their signature — the \
|
||
registry keeps one row per name, so one account is being discarded: {}",
|
||
dup_conflicts.len(),
|
||
dup_conflicts.join(", ")
|
||
);
|
||
}
|
||
}
|
||
eprintln!(
|
||
" entity-IO handlers class-qualified by their array's own field descriptors: {dd_qualified}/{dd_total} \
|
||
(the schema and the datadesc are independent reflection systems; an array whose fingerprint fits \
|
||
several classes or none names neither)"
|
||
);
|
||
println!(
|
||
" binding registry: {} Pulse bindings, {} entity-IO inputs, {} outputs, {} entity classnames",
|
||
out.bindings.meta.pulse,
|
||
out.bindings.meta.entity_inputs,
|
||
out.bindings.meta.entity_outputs,
|
||
out.bindings.meta.entity_classes
|
||
);
|
||
out
|
||
}
|
||
|
||
/// The Valve-table stage's products beyond the locators it folds into `t3`.
|
||
pub(crate) struct ValveTables {
|
||
/// Argument footprints measured for the names it folded, for the monolith's `abi` field.
|
||
abi: BTreeMap<String, model::AbiShape>,
|
||
/// The declared callable surface, shipped as its own artifact.
|
||
bindings: model::Bindings,
|
||
}
|
||
|
||
/// The receiver shape a Pulse binding's flags describe. The two bytes are mutually exclusive in the
|
||
/// data; `Cell` is the neither case — a cell entry point the VM drives, not a graph-author binding.
|
||
fn binding_kind(f: valvetab::PulseFlags) -> model::BindingKind {
|
||
match (f.library, f.instance) {
|
||
(true, false) => model::BindingKind::Library,
|
||
(false, true) => model::BindingKind::Instance,
|
||
_ => model::BindingKind::Cell,
|
||
}
|
||
}
|
||
|
||
/// Fold string anchors onto every monolith entry whose name carries one, in every tier.
|
||
///
|
||
/// Returns how many landed. That number is REPORTED rather than assumed because the two populations are
|
||
/// independent: the catalogue says which names have anchors, the derive says which names got a locator, and
|
||
/// an anchor for a name that never resolved has nowhere to go. A large gap is a fact about the build, not a
|
||
/// bug — but it should be visible rather than inferred from an artifact diff.
|
||
fn attach_anchors(mono: &mut model::Monolith, anchors: &BTreeMap<String, Vec<String>>) -> usize {
|
||
let mut n = 0;
|
||
for tier in [
|
||
&mut mono.core,
|
||
&mut mono.high_confidence,
|
||
&mut mono.experimental,
|
||
] {
|
||
for (name, e) in tier.iter_mut() {
|
||
if let Some(a) = anchors.get(name) {
|
||
// Deduplicated on the way in: the same anchor can appear on several catalogue variants,
|
||
// and this list ships in a byte-reproducible artifact.
|
||
for s in a {
|
||
if !e.locator.anchors.contains(s) {
|
||
e.locator.anchors.push(s.clone());
|
||
}
|
||
}
|
||
n += 1;
|
||
}
|
||
}
|
||
}
|
||
n
|
||
}
|
||
|
||
/// A string worth anchoring on: long enough to be distinctive, printable, and not a lone format specifier.
|
||
///
|
||
/// The thresholds are the knob this whole feature turns on. Loosening them raises coverage and lowers
|
||
/// distinctiveness; they were measured, not guessed — at these values 4,322 of libserver's 70,288 functions
|
||
/// have a unique anchor, and 27% of the shipped set does.
|
||
fn usable_anchor(s: &str) -> bool {
|
||
s.len() >= 8
|
||
&& s.len() <= 200
|
||
&& s.is_ascii()
|
||
&& s.chars().filter(|c| c.is_ascii_alphanumeric()).count() >= 5
|
||
}
|
||
|
||
/// Where every anchorable string in `img` is referenced FROM: string address -> the instruction addresses
|
||
/// that load it, plus the string itself.
|
||
///
|
||
/// Deliberately instruction-level and function-agnostic. Attributing a string to a function needs function
|
||
/// BOUNDARIES, and this binary does not reliably supply them — `.eh_frame_hdr` describes 8,327 of libserver's
|
||
/// ~70,000 functions, so a `[entry, next_entry)` range routinely spans a real function plus one or more
|
||
/// unindexed neighbours, and every neighbour's strings then look like the first function's. An instruction
|
||
/// address, by contrast, is exactly what it is. The caller decides membership against an extent it walked
|
||
/// itself, which is the only claim available that does not depend on the entry list being complete.
|
||
fn string_refs(img: &CodeImage) -> HashMap<u64, (String, Vec<u64>)> {
|
||
let entries = crate::locate::function_entries(img);
|
||
|
||
let mut out: HashMap<u64, (String, Vec<u64>)> = HashMap::new();
|
||
for (i, &start) in entries.iter().enumerate() {
|
||
let end = entries.get(i + 1).copied().unwrap_or(u64::MAX);
|
||
let Some(code) = img.code_range(start, end) else {
|
||
continue;
|
||
};
|
||
let mut insn = iced_x86::Instruction::default();
|
||
let mut dec = iced_x86::Decoder::with_ip(64, code, start, iced_x86::DecoderOptions::NONE);
|
||
while dec.can_decode() {
|
||
dec.decode_out(&mut insn);
|
||
if insn.is_invalid() || !insn.is_ip_rel_memory_operand() {
|
||
continue;
|
||
}
|
||
let va = insn.ip_rel_memory_address();
|
||
if let Some(e) = out.get_mut(&va) {
|
||
e.1.push(insn.ip());
|
||
} else if let Some(s) = img.read_c_string(va).filter(|s| usable_anchor(s)) {
|
||
out.insert(va, (s, vec![insn.ip()]));
|
||
}
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Instruction addresses reachable from `entry` by following control flow, and the string addresses it loads.
|
||
///
|
||
/// The function's OWN extent, determined by where its branches go and where it returns, rather than by the
|
||
/// next symbol. That is what makes the anchor check sound without a complete function list.
|
||
fn reachable_strings(img: &CodeImage, entry: u64) -> Option<(HashSet<u64>, Vec<u64>)> {
|
||
const CAP: u64 = 0x4000;
|
||
let all = img.code_at(entry)?;
|
||
let extent = (all.len() as u64).min(CAP);
|
||
let code = &all[..extent as usize];
|
||
let mut seen: HashSet<u64> = HashSet::new();
|
||
let mut loads: Vec<u64> = Vec::new();
|
||
// Saturating for the same reason `pulse::shim_reads` is: a file-controlled extent must not wrap the
|
||
// range inside out under the overflow-checked build the fuzzers use.
|
||
let end = entry.saturating_add(extent);
|
||
let mut work = vec![entry];
|
||
let mut insn = iced_x86::Instruction::default();
|
||
while let Some(at) = work.pop() {
|
||
if at < entry || at >= end || !seen.insert(at) || seen.len() > 40000 {
|
||
continue;
|
||
}
|
||
let mut dec = iced_x86::Decoder::with_ip(
|
||
64,
|
||
&code[(at - entry) as usize..],
|
||
at,
|
||
iced_x86::DecoderOptions::NONE,
|
||
);
|
||
if !dec.can_decode() {
|
||
continue;
|
||
}
|
||
dec.decode_out(&mut insn);
|
||
if insn.is_invalid() || insn.len() == 0 {
|
||
continue;
|
||
}
|
||
if insn.is_ip_rel_memory_operand() {
|
||
loads.push(insn.ip_rel_memory_address());
|
||
}
|
||
match insn.flow_control() {
|
||
iced_x86::FlowControl::Return
|
||
| iced_x86::FlowControl::IndirectBranch
|
||
| iced_x86::FlowControl::Exception
|
||
| iced_x86::FlowControl::Interrupt => {}
|
||
iced_x86::FlowControl::UnconditionalBranch => work.push(insn.near_branch_target()),
|
||
iced_x86::FlowControl::ConditionalBranch => {
|
||
work.push(at + insn.len() as u64);
|
||
work.push(insn.near_branch_target());
|
||
}
|
||
_ => work.push(at + insn.len() as u64),
|
||
}
|
||
}
|
||
Some((seen, loads))
|
||
}
|
||
|
||
/// Derive an anchor for every entry that has none, from the address its SHIPPED signature resolves to.
|
||
///
|
||
/// Three conditions, each closing a way this can name the wrong function:
|
||
///
|
||
/// 1. **The resolved address must be a function ENTRY POINT.** ModSharp's `refs.strings` locates a
|
||
/// *function*; a great many shipped locators deliberately point MID-function (`CBaseButton::InputPress`
|
||
/// resolves to a `mov`, `BotNavIgnore` to a `je` — patterns anchored at a hook site, not a prologue). An
|
||
/// anchor cannot denote the same thing as one of those, so those entries get none rather than a locator
|
||
/// that resolves somewhere else.
|
||
/// 2. **The string must be referenced from inside the function's OWN flow-reachable code**, walked from the
|
||
/// entry, not from a `[entry, next_entry)` range. `.eh_frame_hdr` covers a small fraction of these
|
||
/// binaries' functions, so such a range routinely swallows unindexed neighbours and inherits their
|
||
/// strings — which is exactly how a first cut of this produced "`CBaseButton::InputPress` references
|
||
/// *Traced intervals in %.3fus*".
|
||
/// 3. **Every instruction that loads the string must be inside that same reachable set.** This is the
|
||
/// uniqueness test, done at instruction level so it never consults a function boundary. A string also
|
||
/// loaded from elsewhere locates nothing and is dropped.
|
||
///
|
||
/// Server-library only, the restriction [`recover_by_string_anchor`] already carries: the fold holds that one
|
||
/// image, and loading a second full set of 22 keyed images to reach the rest would add several hundred MB to a
|
||
/// pipeline that has already been OOM-killed on Dota.
|
||
///
|
||
/// Returns `(attached, considered)`. Degrades quietly toward FEWER anchors and never toward a wrong one.
|
||
fn attach_derived_anchors(
|
||
mono: &mut model::Monolith,
|
||
img: &CodeImage,
|
||
server_lib: &str,
|
||
) -> (usize, usize) {
|
||
let wants = |e: &model::MonoEntry| {
|
||
e.locator.anchors.is_empty()
|
||
&& e.locator
|
||
.signature
|
||
.as_ref()
|
||
.is_some_and(|s| s.library == server_lib)
|
||
};
|
||
let any = [&mono.core, &mono.high_confidence, &mono.experimental]
|
||
.iter()
|
||
.any(|t| t.values().any(&wants));
|
||
if !any {
|
||
return (0, 0);
|
||
}
|
||
let refs = string_refs(img);
|
||
// Condition 1's test set: the addresses this image treats as function starts.
|
||
let starts = crate::locate::function_entries(img);
|
||
|
||
// Why each candidate was rejected, so a low yield is a FACT rather than a mystery. The three
|
||
// conditions fail for very different reasons and the mix differs sharply between games (CS2 derives
|
||
// ~5% of candidates, Dota ~0.5%); without this the difference is unattributable.
|
||
let (mut attached, mut considered) = (0usize, 0usize);
|
||
let (mut no_resolve, mut mid_fn, mut no_unique) = (0usize, 0usize, 0usize);
|
||
for tier in [
|
||
&mut mono.core,
|
||
&mut mono.high_confidence,
|
||
&mut mono.experimental,
|
||
] {
|
||
for e in tier.values_mut() {
|
||
if !wants(e) {
|
||
continue;
|
||
}
|
||
considered += 1;
|
||
let Some(sig) = e.locator.signature.as_ref() else {
|
||
continue;
|
||
};
|
||
let Ok(pat) = crate::sig::Pattern::parse(&sig.linux) else {
|
||
continue;
|
||
};
|
||
let hits = img.find(&pat);
|
||
let [addr] = hits.as_slice() else {
|
||
no_resolve += 1;
|
||
continue;
|
||
};
|
||
let addr = *addr;
|
||
if starts.binary_search(&addr).is_err() {
|
||
mid_fn += 1; // condition 1: mid-function locator, not a function an anchor can name
|
||
continue;
|
||
}
|
||
let Some((reach, loads)) = reachable_strings(img, addr) else {
|
||
continue;
|
||
};
|
||
// Candidates this function actually loads, longest first then lexicographic — deterministic,
|
||
// because this lands in a byte-reproducible artifact.
|
||
let mut cands: Vec<&(String, Vec<u64>)> = loads
|
||
.iter()
|
||
.filter_map(|va| refs.get(va))
|
||
.filter(|(_, from)| from.iter().all(|ip| reach.contains(ip)))
|
||
.collect();
|
||
cands.sort_unstable_by(|a, b| b.0.len().cmp(&a.0.len()).then_with(|| a.0.cmp(&b.0)));
|
||
match cands.first() {
|
||
Some((s, _)) => {
|
||
e.locator.anchors.push(s.clone());
|
||
attached += 1;
|
||
}
|
||
None => no_unique += 1,
|
||
}
|
||
}
|
||
}
|
||
eprintln!(
|
||
" anchor derivation: {attached} attached; rejected {no_resolve} (pattern did not resolve \
|
||
uniquely), {mid_fn} (locator is mid-function, which an anchor cannot name), {no_unique} (no string \
|
||
unique to the function)"
|
||
);
|
||
(attached, considered)
|
||
}
|
||
|
||
/// Assemble the monolith in memory and render its CS# gamedata (the string the live validate stage checks).
|
||
/// Writes NOTHING — `produce` holds the `Monolith` (to annotate it live) plus this render, and writes the
|
||
/// monolith exactly once at the end (after live validation, if a game is present).
|
||
fn build_monolith(
|
||
prof: &GameProfile,
|
||
source_build: &str,
|
||
version: &str,
|
||
core_json: &str,
|
||
flagged: &[model::Flagged],
|
||
experimental: Option<&[ExpGuess]>,
|
||
unverified: &BTreeSet<String>,
|
||
t3: &BTreeMap<String, model::Entry>,
|
||
prov: &BTreeMap<String, model::Provenance>,
|
||
abi: &BTreeMap<String, model::AbiShape>,
|
||
anchors: &BTreeMap<String, Vec<String>>,
|
||
server_img: Option<(&CodeImage, &str)>,
|
||
) -> Result<(model::Monolith, String)> {
|
||
let mut mono = assemble_monolith(
|
||
prof,
|
||
source_build,
|
||
version,
|
||
core_json,
|
||
flagged,
|
||
experimental,
|
||
unverified,
|
||
t3,
|
||
prov,
|
||
abi,
|
||
)?;
|
||
// Attached AFTER assembly, across every tier at once, rather than at the three MonoEntry construction
|
||
// sites: an anchor belongs to a NAME, not to a tier, and one pass cannot leave a tier out by omission.
|
||
let attached = attach_anchors(&mut mono, anchors);
|
||
// Then DERIVE one for everything the catalogue does not cover. Reported separately from the catalogued
|
||
// count: they are different claims — one is a curated string somebody chose, the other is this build's
|
||
// own machine code answering the same question — and collapsing them would hide either going to zero.
|
||
let (derived, considered) = match server_img {
|
||
Some((img, lib)) => attach_derived_anchors(&mut mono, img, lib),
|
||
None => (0, 0),
|
||
};
|
||
eprintln!(
|
||
" string anchors: {attached} of {} catalogued reached the monolith; {derived} DERIVED for {considered} \
|
||
server entries that had none ({} total anchored)",
|
||
anchors.len(),
|
||
attached + derived
|
||
);
|
||
let cssharp = render::render_monolith_cssharp(&mono, model::TierSelect::HighConfidence);
|
||
eprintln!(
|
||
" monolith: {} core + {} high-conf + {} experimental + {} unresolved",
|
||
mono.meta.counts.core,
|
||
mono.meta.counts.high_confidence,
|
||
mono.meta.counts.experimental,
|
||
mono.meta.counts.unresolved
|
||
);
|
||
eprintln!(
|
||
" aliases: {} names over {} functions carry a second shipped name ({:.1}% of the resolved \
|
||
surface is one function under several names)",
|
||
mono.meta.aliased_names,
|
||
mono.meta.alias_groups,
|
||
100.0 * mono.meta.aliased_names as f64
|
||
/ (mono.meta.counts.core + mono.meta.counts.high_confidence).max(1) as f64
|
||
);
|
||
Ok((mono, cssharp))
|
||
}
|
||
|
||
/// The fold's inputs, grouped so a new fold input is added in ONE place — not threaded through a long
|
||
/// positional call where two same-typed `Option<&Path>` are a silent transposition hazard. Built by the
|
||
/// `produce` orchestration; the `--extra-*`/`--full-names` members are the multilib/experimental opt-ins.
|
||
pub(crate) struct FoldArgs<'a> {
|
||
pub build: &'a Path,
|
||
pub lib: &'a str,
|
||
/// Optional bring-up refinements (which extrapolated names may promote, and their prefiltered
|
||
/// context). `None` = empty; a game derives its full catalogue without either.
|
||
pub promotable: Option<&'a Path>,
|
||
pub candidates: Option<&'a Path>,
|
||
/// The derive's in-memory rendered core gamedata + its flag list (`None` when empty) — handed straight
|
||
/// from `gamedata()`, no disk round-trip.
|
||
pub core: &'a str,
|
||
pub flagged: &'a [model::Flagged],
|
||
/// Core names whose fingerprint check rejected every candidate (see [`Derived::unverified`]) — marked
|
||
/// `catalogue-unverified` in the monolith rather than passed off as fingerprint-verified.
|
||
pub unverified: &'a BTreeSet<String>,
|
||
/// The derive's measured argument footprints, folded onto the monolith entries.
|
||
pub abi: &'a BTreeMap<String, model::AbiShape>,
|
||
/// The derive's catalogue string anchors, folded onto the monolith entries the same way.
|
||
pub anchors: &'a BTreeMap<String, Vec<String>>,
|
||
pub sig_cap: usize,
|
||
pub version: &'a str,
|
||
pub full_names: Option<&'a Path>,
|
||
pub extra_offsets: Option<&'a Path>,
|
||
pub extra_sigs: Option<&'a Path>,
|
||
/// Valve's naming for the entity class behind each `PVAL_EHANDLE` parameter
|
||
/// (`mappings/ehandle-classes.json`). Static repo input, optional: without it the bindings artifact
|
||
/// states no class, which is what it stated before this existed.
|
||
pub ehandle_classes: Option<&'a Path>,
|
||
pub source_build: &'a str,
|
||
}
|
||
|
||
/// Measure the argument footprint of OFFSET-located entries. A vtable slot IS the function address once the
|
||
/// class's vtable is located, so the same `abi_shape` that covers signatures covers virtuals — and virtuals
|
||
/// are the bulk of the shipped surface. Signature entries already carry a shape from the derive; this fills
|
||
/// the other half.
|
||
///
|
||
/// Searches EVERY server-mapped library, not just the default one: an extrapolated name routinely sits on a
|
||
/// class owned by `engine2`/`tier0`/etc., and looking only in the game lib silently skips those. Vtables are
|
||
/// resolved once per class, since one class backs many entries.
|
||
fn offset_abi_shapes<'a>(
|
||
prof: &GameProfile,
|
||
imgs: &[CodeImage],
|
||
entries: impl Iterator<Item = (&'a str, &'a str, i64)>,
|
||
) -> BTreeMap<String, model::AbiShape> {
|
||
let mut out = BTreeMap::new();
|
||
let mut vts: HashMap<String, Option<(usize, rtti::VTable)>> = HashMap::new();
|
||
for (name, class, slot) in entries {
|
||
if class.is_empty() || class == name {
|
||
continue; // a free function has no vtable to index
|
||
}
|
||
let hit = vts.entry(class.to_string()).or_insert_with(|| {
|
||
imgs.iter().enumerate().find_map(|(i, im)| {
|
||
rtti::find_vtable(im, class, prof.max_vtable_slots).map(|vt| (i, vt))
|
||
})
|
||
});
|
||
let (Some((li, vt)), Ok(idx)) = (hit.as_ref(), usize::try_from(slot)) else {
|
||
continue;
|
||
};
|
||
if let Some(&addr) = vt.slots.get(idx)
|
||
&& let Some(sh) = abi::abi_shape(&imgs[*li], addr)
|
||
{
|
||
out.insert(name.to_string(), shipped_abi(sh));
|
||
}
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Fold the verified name-extrapolation harvest + curated entries under the guaranteed core into ONE
|
||
/// shipped gamedata — the whole deterministic promote stage in a single per-game command. Each promotable
|
||
/// gets the correct-by-construction locator (a vtable OFFSET when its RTTI class is clean and the AI's
|
||
/// class matches ground-truth, else a fresh `make_sig`); the prefiltered `--candidates` membership drops
|
||
/// dead weight; the guaranteed `--core` wins name collisions. `--extra-offsets` folds multilib ground-truth
|
||
/// vtable methods (name + slot) from the other server libs directly as high_confidence offsets. Emits the
|
||
/// combined file + a provenance sidecar (tier / confidence / return-class / by-value flag).
|
||
pub(crate) fn build_gamedata_cmd(prof: &GameProfile, a: FoldArgs) -> Result<Folded> {
|
||
let FoldArgs {
|
||
build,
|
||
lib,
|
||
promotable,
|
||
candidates,
|
||
core,
|
||
flagged,
|
||
unverified,
|
||
abi,
|
||
anchors,
|
||
sig_cap,
|
||
version,
|
||
full_names,
|
||
extra_offsets,
|
||
extra_sigs,
|
||
ehandle_classes,
|
||
source_build,
|
||
} = a;
|
||
let img = load_lib(build, lib)?;
|
||
let default_lib = lib_name_from_file(lib);
|
||
|
||
// Both are OPTIONAL bring-up inputs: they refine which extrapolated names get promoted, and a game
|
||
// with neither still derives its whole catalogue. Absent = empty, so standing up a new game needs only
|
||
// a catalogue rather than two files with no generator in the repo.
|
||
let promo: Vec<PromoName> = match promotable {
|
||
Some(p) => {
|
||
serde_json::from_str(&std::fs::read_to_string(p)?).context("parse promotable json")?
|
||
}
|
||
None => Vec::new(),
|
||
};
|
||
let cdoc: CandDoc = match candidates {
|
||
Some(p) => {
|
||
serde_json::from_str(&std::fs::read_to_string(p)?).context("parse candidates json")?
|
||
}
|
||
None => CandDoc::default(),
|
||
};
|
||
let cand: HashMap<String, CandCtx> = cdoc
|
||
.candidates
|
||
.into_iter()
|
||
.map(|c| (c.addr.clone(), c))
|
||
.collect();
|
||
|
||
// Protobuf message types are dead weight even when the class name (e.g. `AccountActivity`,
|
||
// `CGCToGCMsgMasterAck`) matches no foreign/CMsg prefix — only the serializer METHOD names reveal them.
|
||
// Cluster over the promoted set (the names this fold would ship).
|
||
let protobuf_classes = protobuf_message_classes(
|
||
prof,
|
||
promo.iter().filter(|p| p.promote).map(|p| p.name.as_str()),
|
||
);
|
||
|
||
let mut t3: BTreeMap<String, model::Entry> = BTreeMap::new();
|
||
let mut prov: BTreeMap<String, model::Provenance> = BTreeMap::new();
|
||
|
||
// Valve's own in-binary name tables, folded FIRST: the extrapolation loop below skips any name `t3`
|
||
// already holds, so folding here is what ranks ground truth from this binary above every inferred
|
||
// name. (`core` still wins everything — its overlap removal runs after both.)
|
||
let mut valve = fold_valve_tables(prof, build, source_build, sig_cap, &mut t3, &mut prov);
|
||
|
||
// Valve's own naming for the entity class behind each handle, propagated across the parameters this
|
||
// build proves are the same type. A static repo input like `mappings/prototypes.json` — INPUT, never
|
||
// folded forward — and optional: without it the artifact simply states no class, which is what it
|
||
// stated before this existed.
|
||
if let Some(path) = ehandle_classes {
|
||
let table: EhandleClasses = serde_json::from_str(
|
||
&std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?,
|
||
)
|
||
.context("parse ehandle-classes json")?;
|
||
let (named, propagated, conflicts) =
|
||
name_ehandle_classes(&mut valve.bindings.pulse, &table);
|
||
println!(
|
||
" PVAL_EHANDLE entity classes: {named} named from Valve's metadata, {propagated} more \
|
||
propagated across parameters the destructor address proves are the same type"
|
||
);
|
||
// Two-sided: the names come from Valve's dump, the grouping from the binary's destructor
|
||
// addresses, and neither reader knows the other exists. Agreement is evidence for both.
|
||
if conflicts.is_empty() {
|
||
println!(" PVAL_EHANDLE grouping vs Valve's naming: no group carries two classes");
|
||
} else {
|
||
for c in &conflicts {
|
||
println!(" EHANDLE CONFLICT {c}");
|
||
}
|
||
}
|
||
}
|
||
|
||
let (mut n_off, mut n_sig, mut n_byval) = (0u32, 0u32, 0u32);
|
||
let (mut dead, mut unmakeable) = (0u32, 0u32);
|
||
let mut unmakeable_list = Vec::new();
|
||
|
||
// The fold's make_sig dominates the cost and is a pure read on `img`, so do it ONCE in
|
||
// parallel for every promoted addr that routes to a SIGNATURE (not a vtable offset); the loop
|
||
// below just looks the result up. make_sig is a pure fn of (img, addr, cap), so the fold's
|
||
// output is byte-identical to the serial version — this changes WHERE the work runs, not what
|
||
// it produces. (Inherited methods, whose declaring class != the candidate's RTTI class, fall to
|
||
// make_sig rather than an offset, so this path carries real load.)
|
||
let sig_addrs: Vec<u64> = promo
|
||
.iter()
|
||
.filter(|pr| {
|
||
pr.promote
|
||
&& !is_dead_weight_name(prof, &pr.name)
|
||
&& !is_serializer_plumbing(prof, &pr.name, &protobuf_classes)
|
||
})
|
||
.filter_map(|pr| {
|
||
let c = cand.get(&pr.addr)?;
|
||
if is_offset_locator(prof, pr, c) {
|
||
return None;
|
||
}
|
||
parse_hex_addr(&pr.addr)
|
||
})
|
||
.collect::<HashSet<u64>>()
|
||
.into_iter()
|
||
.collect();
|
||
let sig_by_addr: HashMap<u64, String> = parallel_map(&sig_addrs, default_threads(None), |&a| {
|
||
(a, emit::make_sig(&img, a, sig_cap))
|
||
})
|
||
.into_iter()
|
||
.filter_map(|(a, s)| s.map(|s| (a, s)))
|
||
.collect();
|
||
|
||
for pr in &promo {
|
||
if !pr.promote || t3.contains_key(&pr.name) {
|
||
continue;
|
||
}
|
||
// Dead-weight gates: (1) the resolved NAME (foreign/protobuf functions the class-based dump
|
||
// prefilter can't see); (2) a lone HARD serializer method or a protobuf message class (>=3
|
||
// serializer methods); (3) membership in the prefiltered candidate dump.
|
||
if is_dead_weight_name(prof, &pr.name)
|
||
|| is_serializer_plumbing(prof, &pr.name, &protobuf_classes)
|
||
{
|
||
dead += 1;
|
||
continue;
|
||
}
|
||
let Some(c) = cand.get(&pr.addr) else {
|
||
dead += 1;
|
||
continue;
|
||
};
|
||
let ret = ByValClass::from_abi(c.abi.as_deref());
|
||
if is_offset_locator(prof, pr, c) {
|
||
t3.insert(pr.name.clone(), model::Entry::offset(c.slot.unwrap()));
|
||
n_off += 1;
|
||
} else {
|
||
let Some(addr) = parse_hex_addr(&pr.addr) else {
|
||
unmakeable += 1;
|
||
unmakeable_list.push(format!("{} (bad addr {:?})", pr.name, pr.addr));
|
||
continue;
|
||
};
|
||
match sig_by_addr.get(&addr).cloned() {
|
||
Some(sig) => {
|
||
t3.insert(
|
||
pr.name.clone(),
|
||
model::Entry::signature(default_lib.to_string(), sig),
|
||
);
|
||
n_sig += 1;
|
||
}
|
||
None => {
|
||
unmakeable += 1;
|
||
unmakeable_list.push(format!("{} @ {addr:#x}", pr.name));
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
let tier = if pr.self_named {
|
||
model::Tier::SelfNamed
|
||
} else if pr.corroboration == "exact" {
|
||
model::Tier::DictExact
|
||
} else {
|
||
model::Tier::Contextual
|
||
};
|
||
if ret.is_by_value() {
|
||
n_byval += 1;
|
||
}
|
||
// Build the provenance TYPED so the compiler enforces the schema (model::Provenance serializes its
|
||
// fields in declaration order).
|
||
prov.insert(
|
||
pr.name.clone(),
|
||
model::Provenance {
|
||
addr: Some(pr.addr.clone()),
|
||
confidence: Some(pr.confidence.clone()),
|
||
self_named: Some(pr.self_named),
|
||
rationale: Some(pr.rationale.clone()),
|
||
ret_class: Some(format!("ret={}", ret.as_str())),
|
||
by_value: Some(ret.is_by_value()),
|
||
source: Some("source2rosetta-nameext".to_string()),
|
||
rtti_class: c.class.clone(),
|
||
..model::Provenance::with_tier(tier)
|
||
},
|
||
);
|
||
}
|
||
|
||
// fold under the guaranteed core (core wins any name collision) — parsed from the derive's in-memory
|
||
// rendered core, no disk round-trip.
|
||
let core_gd: GdMap = read_gamedata_str(core)?;
|
||
let overlaps: Vec<String> = t3
|
||
.keys()
|
||
.filter(|k| core_gd.contains_key(*k))
|
||
.cloned()
|
||
.collect();
|
||
for k in &overlaps {
|
||
t3.remove(k);
|
||
prov.remove(k);
|
||
}
|
||
|
||
// Multilib ground-truth folds for the non-primary libs (macOS symbol-transfer offsets + AI-name sigs),
|
||
// each under the same core-wins precedence — see fold_extra_offsets / fold_extra_sigs.
|
||
if let Some(ep) = extra_offsets {
|
||
fold_extra_offsets(ep, &core_gd, &mut t3, &mut prov)?;
|
||
}
|
||
if let Some(sp) = extra_sigs {
|
||
fold_extra_sigs(build, sp, sig_cap, &core_gd, &mut t3, &mut prov)?;
|
||
}
|
||
|
||
let tier_count = |t: model::Tier| prov.values().filter(|p| p.tier == t).count();
|
||
let (n_self, n_dict) = (
|
||
tier_count(model::Tier::SelfNamed),
|
||
tier_count(model::Tier::DictExact),
|
||
);
|
||
|
||
eprintln!(
|
||
"produce (fold) {version}: {} core + {} extrapolated = {} functions ({n_off} offsets, {n_sig} sigs; \
|
||
{n_self} self-named, {n_dict} dict-exact, {n_byval} by-value-flagged)",
|
||
core_gd.len(),
|
||
t3.len(),
|
||
core_gd.len() + t3.len()
|
||
);
|
||
eprintln!(
|
||
" {dead} promotable dropped (dead-weight, absent from prefiltered candidates); {unmakeable} unmakeable-sig; {} core-name overlaps dropped",
|
||
overlaps.len()
|
||
);
|
||
for u in unmakeable_list.iter().take(20) {
|
||
eprintln!(" UNMAKEABLE {u}");
|
||
}
|
||
|
||
// Least-filtered inclusion tier: the experimental band, in memory, emitted iff `--full-names` is given.
|
||
// Emitted here (the fold step) because this is where names + locators become gamedata entries — the SAME
|
||
// derivation, just with no promote/dead-weight gate.
|
||
let experimental = full_names
|
||
.map(|fnames| emit_experimental_band(prof, &img, build, fnames, &default_lib, sig_cap))
|
||
.transpose()?;
|
||
|
||
// The derive measured the signature entries; virtuals are measured HERE, where the target images are
|
||
// loaded and a slot resolves to an address. Together they cover the whole shipped surface.
|
||
let abi_imgs = load_build_images(prof, build);
|
||
let core_doc: BTreeMap<String, Value> =
|
||
serde_json::from_str(core).context("parse core gamedata for ABI shapes")?;
|
||
// THE RULE FOR THIS MAP: it is keyed by NAME, and `assemble_monolith` attaches each shape to the entry
|
||
// of that name — so every shape in it must have been measured at the address that entry's SHIPPED
|
||
// locator resolves to. A shape measured at any other address describes a different function, and would
|
||
// be published as this one's argument footprint (which is also what makes a declared prototype
|
||
// checkable, so a wrong one silently converts a correct declaration into a "mismatch").
|
||
//
|
||
// The Valve-table stage measured its entries as it located them, but the core-overlap removal above may
|
||
// since have taken a name away from it — drop those, or the shape would outlive the locator it belongs to.
|
||
let mut abi_all = valve.abi;
|
||
abi_all.retain(|name, _| t3.contains_key(name));
|
||
// The derive's own measurements win on any shared name, being taken through the locator that ships.
|
||
abi_all.extend(abi.iter().map(|(k, v)| (k.clone(), v.clone())));
|
||
// via the canonical inverse, not a hand-rolled key lookup: the rendered core nests its locator
|
||
// (`offsets.linux`), and `entry_from_value` is the one place that shape is known.
|
||
let core_offsets: Vec<(&str, String, i64)> = core_doc
|
||
.iter()
|
||
.filter_map(|(n, v)| {
|
||
Some((
|
||
n.as_str(),
|
||
class_of(n).to_string(),
|
||
render::entry_from_value(v).offset?,
|
||
))
|
||
})
|
||
.collect();
|
||
abi_all.extend(offset_abi_shapes(
|
||
prof,
|
||
&abi_imgs,
|
||
core_offsets.iter().map(|(n, c, o)| (*n, c.as_str(), *o)),
|
||
));
|
||
let t3_offsets: Vec<(&str, String, i64)> = t3
|
||
.iter()
|
||
.filter_map(|(n, e)| Some((n.as_str(), class_of(n).to_string(), e.offset?)))
|
||
.collect();
|
||
abi_all.extend(offset_abi_shapes(
|
||
prof,
|
||
&abi_imgs,
|
||
t3_offsets.iter().map(|(n, c, o)| (*n, c.as_str(), *o)),
|
||
));
|
||
// The experimental band carries its class explicitly (a guessed name need not parse as Class::Method).
|
||
// A guess is measured at ITS OWN address — a different function from the one a shipped entry of the same
|
||
// name locates — so names owned by `core` or by the fold are excluded, mirroring `assemble_monolith`,
|
||
// which drops the guess itself for exactly this reason. This extend runs LAST, so without the guard a
|
||
// guess would overwrite the shipped entry's footprint and nothing downstream would mark the swap.
|
||
if let Some(gs) = experimental.as_deref() {
|
||
let exp: Vec<(&str, &str, i64)> = gs
|
||
.iter()
|
||
.filter(|g| !g.promoted && !core_gd.contains_key(&g.name) && !t3.contains_key(&g.name))
|
||
.filter_map(|g| {
|
||
Some((
|
||
g.name.as_str(),
|
||
g.class.as_deref()?,
|
||
g.locator.as_ref()?.offset?,
|
||
))
|
||
})
|
||
.collect();
|
||
abi_all.extend(offset_abi_shapes(prof, &abi_imgs, exp.into_iter()));
|
||
}
|
||
|
||
// Assemble the monolith (high_confidence from the fold's own `t3`/`prov`; core/unresolved/experimental
|
||
// from the derive's in-memory strings) + its CS# render, returned to `produce` — nothing written here.
|
||
let (mono, cssharp) = build_monolith(
|
||
prof,
|
||
source_build,
|
||
version,
|
||
core,
|
||
flagged,
|
||
experimental.as_deref(),
|
||
unverified,
|
||
&t3,
|
||
&prov,
|
||
&abi_all,
|
||
anchors,
|
||
Some((&img, &default_lib)),
|
||
)?;
|
||
Ok(Folded {
|
||
mono,
|
||
cssharp,
|
||
bindings: valve.bindings,
|
||
})
|
||
}
|
||
|
||
/// What the fold hands back to `produce`: the monolith, the CS# render live validation checks against,
|
||
/// and the binding registry read out of the same libraries on the way past.
|
||
pub(crate) struct Folded {
|
||
pub mono: model::Monolith,
|
||
pub cssharp: String,
|
||
pub bindings: model::Bindings,
|
||
}
|
||
|
||
/// One experimental-band guess, built by [`emit_experimental_band`] and consumed by [`assemble_monolith`]
|
||
/// into the monolith's experimental tier. Passed as a typed `Vec` in memory — NOT serialized and re-parsed,
|
||
/// so `class` is a typed field a reader can see and `locator` is `None` (not a null sentinel) when the guess
|
||
/// is unlocatable. `promoted` rows are skipped on read (they live in high_confidence).
|
||
struct ExpGuess {
|
||
name: String,
|
||
tier: model::Tier,
|
||
confidence: String,
|
||
corroboration: String,
|
||
self_named: bool,
|
||
promoted: bool,
|
||
collision: bool,
|
||
dead_weight: bool,
|
||
addr: String,
|
||
/// The vtable class an OFFSET guess lives on (experimental-only eyeball aid); `None` for a sig guess.
|
||
class: Option<String>,
|
||
/// The resolved locator, or `None` when the address is neither a vtable slot nor make_sig-able.
|
||
locator: Option<model::Entry>,
|
||
/// The argument footprint measured at the guess's OWN address, here where that address and its image
|
||
/// are both in hand. Re-deriving it later from the class name cannot reach a namespaced or templated
|
||
/// class (`rtti::mangle` has no `_ZTS` needle for `Attribute::Detail::GetterT<...>`), which left 41%
|
||
/// of the CS2 band shipping `abi: null` for functions whose exact address was known here.
|
||
abi: Option<model::AbiShape>,
|
||
}
|
||
|
||
/// Emit the EXPERIMENTAL band — the least-filtered inclusion tier. Every name guess in the full-slice
|
||
/// universe is kept (no promote/dead-weight gate: each row is explicitly a *guess*), each attached to a
|
||
/// locator derived FRESH from this binary — a vtable OFFSET where the address is a virtual slot, else a
|
||
/// `make_sig` SIGNATURE — graded by [`guess_tier`], with same-name collisions flagged and cluster-sorted
|
||
/// so competing guesses sit side by side. One self-describing file under a blunt UNVERIFIED banner.
|
||
/// Locators are derived in parallel (the non-virtual `make_sig` majority dominates the cost).
|
||
fn emit_experimental_band(
|
||
prof: &GameProfile,
|
||
img: &CodeImage,
|
||
build: &Path,
|
||
names_path: &Path,
|
||
default_lib: &str,
|
||
sig_cap: usize,
|
||
) -> Result<Vec<ExpGuess>> {
|
||
// The name universe is either a flat [FullName] array (single-lib → the default lib) or a per-lib
|
||
// { "libX.so": [FullName] } map (multilib). Either shape resolves each guess against its OWN lib image.
|
||
let raw: Value = serde_json::from_str(&std::fs::read_to_string(names_path)?)
|
||
.context("parse full-names json")?;
|
||
let per_lib: BTreeMap<String, Vec<FullName>> = match raw {
|
||
Value::Array(_) => BTreeMap::from([(
|
||
default_lib.to_string(),
|
||
serde_json::from_value(raw).context("parse full-names json")?,
|
||
)]),
|
||
Value::Object(_) => serde_json::from_value(raw).context("parse per-lib full-names json")?,
|
||
_ => {
|
||
anyhow::bail!("full-names json must be a [..] array or a {{ \"lib.so\": [..] }} object")
|
||
}
|
||
};
|
||
|
||
// collision + protobuf clustering span the WHOLE universe (across libs): a name guessed at more than
|
||
// one address — even in different libs — is flagged, and a protobuf class is seen by its full surface.
|
||
let all_rows: Vec<&FullName> = per_lib.values().flatten().collect();
|
||
let mut name_count: HashMap<&str, usize> = HashMap::new();
|
||
for r in &all_rows {
|
||
*name_count.entry(r.name.as_str()).or_default() += 1;
|
||
}
|
||
let pb_classes = protobuf_message_classes(prof, all_rows.iter().map(|r| r.name.as_str()));
|
||
|
||
// resolve each lib's guesses against that lib's OWN image — the default lib reuses the already-loaded
|
||
// `img`; the others load on demand (a guess for a lib absent from this build is skipped). make_sig (the
|
||
// non-virtual majority) dominates the cost, so each lib's batch runs in parallel.
|
||
let mut extra_imgs: HashMap<String, CodeImage> = HashMap::new();
|
||
let mut entries: Vec<(u8, bool, String, String, ExpGuess)> = Vec::new();
|
||
let mut skipped_libs: Vec<String> = Vec::new();
|
||
for (lib_file, rows) in &per_lib {
|
||
let libname = lib_name_from_file(lib_file);
|
||
let limg: &CodeImage = if libname == default_lib {
|
||
img
|
||
} else {
|
||
if !extra_imgs.contains_key(lib_file) {
|
||
let Ok(loaded) = load_lib(build, lib_file) else {
|
||
// a whole library's guesses drop out — report it rather than shrinking the band silently
|
||
skipped_libs.push(format!("{lib_file} ({})", rows.len()));
|
||
continue;
|
||
};
|
||
extra_imgs.insert(lib_file.clone(), loaded);
|
||
}
|
||
&extra_imgs[lib_file]
|
||
};
|
||
|
||
// addr -> (class, slot): the first primary vtable holding the address == a directly callable offset.
|
||
let mut vt_of: HashMap<u64, (String, usize)> = HashMap::new();
|
||
for cv in rtti::enumerate_vtables(limg, prof.max_vtable_slots)
|
||
.into_iter()
|
||
.filter(|c| c.offset_to_top == 0)
|
||
{
|
||
for (i, &a) in cv.slots.iter().enumerate() {
|
||
vt_of.entry(a).or_insert_with(|| (cv.name.clone(), i));
|
||
}
|
||
}
|
||
|
||
let idxs: Vec<usize> = (0..rows.len()).collect();
|
||
let lib_entries: Vec<(u8, bool, String, String, ExpGuess)> =
|
||
parallel_map(&idxs, default_threads(None), |&i| {
|
||
let r = &rows[i];
|
||
let (rank, tier) = guess_tier(r);
|
||
let collision = name_count.get(r.name.as_str()).copied().unwrap_or(0) > 1;
|
||
// protobuf / foreign-namespace / serializer plumbing: kept (nothing hidden) but FLAGGED, so
|
||
// a reader can grep it out and the confidence tiers aren't polluted by generic-method matches
|
||
// (e.g. a protobuf `::Clear` corroborating as "exact" on the dictionary word). Also sunk to
|
||
// the bottom of its tier via the sort key below, so real game functions surface first.
|
||
let dead = is_dead_weight_name(prof, &r.name)
|
||
|| is_serializer_plumbing(prof, &r.name, &pb_classes);
|
||
let addr = parse_hex_addr(&r.addr);
|
||
let abi = addr.and_then(|a| abi::abi_shape(limg, a)).map(shipped_abi);
|
||
let (class, locator): (Option<String>, Option<model::Entry>) =
|
||
match addr.and_then(|a| vt_of.get(&a).map(|v| (a, v))) {
|
||
// a vtable slot: the canonical offset locator + the experimental-only `class` tag
|
||
// (the vtable it lives on) so a reader can eyeball an offset guess.
|
||
Some((_, (cls, slot))) => (
|
||
Some(cls.clone()),
|
||
Some(model::Entry {
|
||
offset: Some(*slot as i64),
|
||
..Default::default()
|
||
}),
|
||
),
|
||
None => match addr.and_then(|a| emit::make_sig(limg, a, sig_cap)) {
|
||
Some(sig) => (
|
||
None,
|
||
Some(model::Entry {
|
||
signature: Some(model::Sig {
|
||
library: libname.clone(),
|
||
linux: sig,
|
||
}),
|
||
..Default::default()
|
||
}),
|
||
),
|
||
None => (None, None),
|
||
},
|
||
};
|
||
let entry = ExpGuess {
|
||
name: r.name.clone(),
|
||
tier,
|
||
confidence: r.confidence.clone(),
|
||
corroboration: r.corroboration.clone(),
|
||
self_named: r.self_named,
|
||
promoted: r.promote,
|
||
collision,
|
||
dead_weight: dead,
|
||
addr: r.addr.clone(),
|
||
class,
|
||
locator,
|
||
abi,
|
||
};
|
||
(rank, dead, r.name.clone(), r.addr.clone(), entry)
|
||
});
|
||
entries.extend(lib_entries);
|
||
}
|
||
|
||
// cluster-sort: tier first (the confidence grouping), then dead-weight LAST within a tier (real game
|
||
// functions surface above the plumbing), then name (collisions become adjacent), then addr.
|
||
entries.sort_by(|a, b| (a.0, a.1, &a.2, &a.3).cmp(&(b.0, b.1, &b.2, &b.3)));
|
||
|
||
let (mut collisions, mut unlocatable, mut dead_weight) = (0usize, 0usize, 0usize);
|
||
let guesses: Vec<ExpGuess> = entries.into_iter().map(|(_, _, _, _, e)| e).collect();
|
||
for e in &guesses {
|
||
collisions += usize::from(e.collision);
|
||
dead_weight += usize::from(e.dead_weight);
|
||
unlocatable += usize::from(e.locator.is_none());
|
||
}
|
||
let total = guesses.len();
|
||
eprintln!(
|
||
"experimental band: {total} guesses ({collisions} name-collisions, {dead_weight} dead-weight-flagged, \
|
||
{unlocatable} unlocatable)"
|
||
);
|
||
if !skipped_libs.is_empty() {
|
||
eprintln!(
|
||
" libs absent from this build (their guesses dropped): {}",
|
||
skipped_libs.join(", ")
|
||
);
|
||
}
|
||
Ok(guesses)
|
||
}
|
||
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// §5 · MONOLITH ASSEMBLY
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
|
||
pub(crate) type GdMap = serde_json::Map<String, serde_json::Value>;
|
||
|
||
// A gamedata entry's locator (a byte SIGNATURE or a vtable OFFSET) is `model::Entry`: construct one with
|
||
// `Entry::signature`/`Entry::offset`, render it with `render::locator_value`, parse it back with
|
||
// `render::entry_from_value` — one type, one write/read pair in `core`, no separate enum or ad-hoc `.get()`.
|
||
|
||
/// Parse a gamedata json that MAY carry `//` line-comment banners, from an in-memory string — the
|
||
/// disk-free path the fold takes for the derive's rendered core and the live validate takes for the
|
||
/// monolith's CS# render, both avoiding a disk round-trip.
|
||
pub(crate) fn read_gamedata_str(raw: &str) -> Result<GdMap> {
|
||
let stripped = raw
|
||
.lines()
|
||
.filter(|l| !l.trim_start().starts_with("//"))
|
||
.collect::<Vec<_>>()
|
||
.join("\n");
|
||
serde_json::from_str(&stripped).context("parse gamedata json")
|
||
}
|
||
|
||
/// Assemble the monolith (the tiered catalogue that becomes `rosetta-<game>.json`'s `functions`) from
|
||
/// the in-memory derive/fold results:
|
||
/// the rendered `core_json` → core tier; the fold's `t3`/`prov` → high_confidence; the `experimental` guesses
|
||
/// → the non-promoted experimental tail; the `flagged` list → unresolved. Pure assembly, no re-derivation and
|
||
/// no intermediate files. `validated` stays `None` (the live stage annotates it later).
|
||
/// Experimental is name-keyed: a collision (one name guessed at several addresses, already best-first in the
|
||
/// input order) collapses to its best representative with `collision:true` retained.
|
||
fn assemble_monolith(
|
||
prof: &GameProfile,
|
||
source_build: &str,
|
||
version: &str,
|
||
core_json: &str,
|
||
flagged: &[model::Flagged],
|
||
experimental: Option<&[ExpGuess]>,
|
||
unverified: &BTreeSet<String>,
|
||
t3: &BTreeMap<String, model::Entry>,
|
||
prov: &BTreeMap<String, model::Provenance>,
|
||
abi: &BTreeMap<String, model::AbiShape>,
|
||
) -> Result<model::Monolith> {
|
||
use model::{Counts, MonoEntry, MonoMeta, Monolith, Provenance, Tier, Unresolved};
|
||
|
||
// core tier: every guaranteed entry in the derive's rendered core, minimal provenance.
|
||
// An entry whose fingerprint check rejected every candidate (emitted only because ONE era-sig resolved
|
||
// uniquely) is marked `catalogue-unverified` — it is in `core` on locator strength alone, and nothing
|
||
// downstream, live validation included, re-checks that it is the right function.
|
||
let mut core: BTreeMap<String, MonoEntry> = BTreeMap::new();
|
||
{
|
||
// ABI prototype-drift detail per name, from the flag list — surfaced onto the core entry (the sig
|
||
// still ships; `assemble` would otherwise drop the abi-drift flag since its name is already in core).
|
||
let abi_drift: BTreeMap<&str, &str> = flagged
|
||
.iter()
|
||
.filter(|f| f.reason == model::FlagReason::AbiDrift)
|
||
.map(|f| (f.name.as_str(), f.detail.as_str()))
|
||
.collect();
|
||
let doc: BTreeMap<String, Value> =
|
||
serde_json::from_str(core_json).context("parse core gamedata")?;
|
||
for (name, v) in &doc {
|
||
let source = if unverified.contains(name) {
|
||
"catalogue-unverified"
|
||
} else {
|
||
"catalogue"
|
||
};
|
||
core.insert(
|
||
name.clone(),
|
||
MonoEntry {
|
||
locator: render::entry_from_value(v),
|
||
abi: abi.get(name.as_str()).cloned(),
|
||
provenance: Provenance {
|
||
source: Some(source.into()),
|
||
abi_drift: abi_drift.get(name.as_str()).map(|d| d.to_string()),
|
||
..Provenance::with_tier(Tier::Core)
|
||
},
|
||
validated: None,
|
||
aliases: Vec::new(), // filled once both shipped tiers exist
|
||
},
|
||
);
|
||
}
|
||
}
|
||
|
||
// high_confidence tier: the promoted names — provenance + locator taken from the IN-MEMORY fold (`t3` =
|
||
// name -> typed `Entry`, `prov` = name -> typed provenance), so no disk round-trip is needed to build the
|
||
// monolith and no untyped re-parse stands between fold and shipped model.
|
||
let mut high: BTreeMap<String, MonoEntry> = BTreeMap::new();
|
||
for (name, provenance) in prov {
|
||
let locator = t3.get(name).cloned().unwrap_or_default();
|
||
high.insert(
|
||
name.clone(),
|
||
MonoEntry {
|
||
locator,
|
||
abi: abi.get(name.as_str()).cloned(),
|
||
provenance: provenance.clone(),
|
||
validated: None,
|
||
aliases: Vec::new(), // filled once both shipped tiers exist
|
||
},
|
||
);
|
||
}
|
||
|
||
// experimental tier: the graded guess tail, EXCLUDING promoted ones (they are high_confidence) and any
|
||
// name already owned by core/high_confidence. First occurrence wins per name (best-first order). Read
|
||
// straight off the typed `Vec<ExpGuess>` — no serialize-and-re-parse, so a renamed field is a compile
|
||
// error here rather than a silent `None`.
|
||
let mut exp_tier: BTreeMap<String, MonoEntry> = BTreeMap::new();
|
||
if let Some(guesses) = experimental {
|
||
for g in guesses {
|
||
if g.promoted {
|
||
continue; // promoted → it lives in high_confidence
|
||
}
|
||
if core.contains_key(&g.name) || high.contains_key(&g.name) {
|
||
continue; // a higher tier already owns this name
|
||
}
|
||
let Some(locator) = &g.locator else {
|
||
continue; // unlocatable guess — not shippable
|
||
};
|
||
// The experimental tier's provenance (the rest of the fields stay None).
|
||
let provenance = Provenance {
|
||
addr: Some(g.addr.clone()),
|
||
confidence: Some(g.confidence.clone()),
|
||
self_named: Some(g.self_named),
|
||
corroboration: Some(g.corroboration.clone()),
|
||
collision: Some(g.collision),
|
||
dead_weight: Some(g.dead_weight),
|
||
..Provenance::with_tier(g.tier)
|
||
};
|
||
exp_tier.entry(g.name.clone()).or_insert(MonoEntry {
|
||
// The class rides on the LOCATOR now, not beside it: for an offset entry the class is what
|
||
// makes the slot index mean anything, and keeping them together is what lets it reach the
|
||
// emitters through `Monolith::select`.
|
||
locator: model::Entry {
|
||
class: g.class.clone(),
|
||
..locator.clone()
|
||
},
|
||
// The band's own measurement first: it was taken at the guess's exact address, which IS
|
||
// what the locator resolves to. The name-keyed map is the fallback.
|
||
abi: g.abi.clone().or_else(|| abi.get(g.name.as_str()).cloned()),
|
||
provenance,
|
||
validated: None,
|
||
// Never grouped: an unverified name sharing a target with another unverified name is not
|
||
// evidence they mean the same thing. This band states the converse through
|
||
// `provenance.collision` (one name guessed at several addresses).
|
||
aliases: Vec::new(),
|
||
});
|
||
}
|
||
}
|
||
|
||
// unresolved tier: catalogue functions the derivation couldn't produce.
|
||
let mut unresolved: BTreeMap<String, Unresolved> = BTreeMap::new();
|
||
for f in flagged {
|
||
// A flagged function can still have been resolved by another path (e.g. a sig-drifted entry
|
||
// recovered as a vtable offset lands in `core`). `unresolved` means truly no locator in any
|
||
// tier, so skip anything a resolved tier already owns — this keeps the four tiers disjoint.
|
||
if core.contains_key(&f.name)
|
||
|| high.contains_key(&f.name)
|
||
|| exp_tier.contains_key(&f.name)
|
||
{
|
||
continue;
|
||
}
|
||
// One name can be flagged by more than one stage (a catalogue entry carrying both a sig and a
|
||
// vtable-offset variant fails each independently). The tier is name-keyed so only one reason
|
||
// fits: keep the FIRST, since the list runs in derivation order and the earlier stage's reason
|
||
// is the more specific one.
|
||
unresolved
|
||
.entry(f.name.clone())
|
||
.or_insert_with(|| Unresolved {
|
||
reason: f.reason.as_str().to_string(),
|
||
detail: f.detail.clone(),
|
||
});
|
||
}
|
||
|
||
let (alias_groups, aliased_names) = link_aliases(&mut core, &mut high);
|
||
|
||
let counts = Counts {
|
||
core: core.len(),
|
||
high_confidence: high.len(),
|
||
experimental: exp_tier.len(),
|
||
unresolved: unresolved.len(),
|
||
};
|
||
Ok(Monolith {
|
||
meta: MonoMeta {
|
||
game_key: prof.game_key.to_string(),
|
||
game: prof.display_name.to_string(),
|
||
source_build: source_build.to_string(),
|
||
version: version.to_string(),
|
||
counts,
|
||
alias_groups,
|
||
aliased_names,
|
||
},
|
||
core,
|
||
high_confidence: high,
|
||
experimental: exp_tier,
|
||
unresolved,
|
||
})
|
||
}
|
||
|
||
/// Cross-link the shipped tiers' ALIASES: names that locate the same function, written onto each entry as
|
||
/// the other names for it. Returns `(groups, names covered)` for the meta counters.
|
||
///
|
||
/// **Locator identity is the key, and for a signature it is exactly address identity.** A shipped pattern is
|
||
/// generated AT the resolved address and then confirmed unique within its library, so two entries carrying
|
||
/// the same `(library, pattern)` cannot resolve anywhere but the same single address — the uniqueness check
|
||
/// the emitter already performs is what makes string equality a sound proxy here, with no second scan. A
|
||
/// vtable entry keys on `(class, slot)`, which is the same argument: the slot means nothing except relative
|
||
/// to a named vtable, and together they name one function.
|
||
///
|
||
/// **A bare slot — an `offset` with no `class` — is deliberately left ungrouped.** It names no vtable, so two
|
||
/// of them sharing a slot index are not evidence of anything; grouping them anyway put 1,080 CS2 names into
|
||
/// 70 fictitious groups when measured. Those entries ship with no `aliases` because none is derivable, not
|
||
/// because none exists, and the field's doc says so rather than letting a reader infer uniqueness from the
|
||
/// silence.
|
||
fn link_aliases(
|
||
core: &mut BTreeMap<String, model::MonoEntry>,
|
||
high: &mut BTreeMap<String, model::MonoEntry>,
|
||
) -> (usize, usize) {
|
||
// The locator's identity as a comparable key, or None where the entry locates nothing groupable.
|
||
let key = |e: &model::MonoEntry| -> Option<(String, String)> {
|
||
if let Some(s) = &e.locator.signature {
|
||
return Some((s.library.clone(), s.linux.clone()));
|
||
}
|
||
match (&e.locator.class, e.locator.offset) {
|
||
(Some(c), Some(slot)) => Some((c.clone(), format!("#{slot}"))),
|
||
_ => None,
|
||
}
|
||
};
|
||
|
||
// Group ACROSS the two tiers, not within each: a `core` name and a `high_confidence` name on one address
|
||
// are aliases, and that pairing is the one a consumer is least likely to spot unaided. A `BTreeSet` per
|
||
// target rather than a `Vec` because the tiers are only disjoint by construction, and a name reaching
|
||
// this twice must not inflate its own group or turn up as its own alias.
|
||
let mut by_target: BTreeMap<(String, String), BTreeSet<String>> = BTreeMap::new();
|
||
for (name, e) in core.iter().chain(high.iter()) {
|
||
if let Some(k) = key(e) {
|
||
by_target.entry(k).or_default().insert(name.clone());
|
||
}
|
||
}
|
||
by_target.retain(|_, names| names.len() > 1);
|
||
|
||
// Resolve every membership first, then write: an entry is reached through whichever tier holds it,
|
||
// and both are written the same way rather than one path shadowing the other.
|
||
let mut others_of: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||
for names in by_target.values() {
|
||
for name in names {
|
||
others_of.insert(
|
||
name.clone(),
|
||
names.iter().filter(|n| *n != name).cloned().collect(),
|
||
);
|
||
}
|
||
}
|
||
for (name, e) in core.iter_mut().chain(high.iter_mut()) {
|
||
if let Some(others) = others_of.get(name) {
|
||
e.aliases = others.clone();
|
||
}
|
||
}
|
||
(by_target.len(), others_of.len())
|
||
}
|
||
|
||
/// Fold the live-validation verdict INTO the monolith (Stage F), from the `verdicts` map the live oracle
|
||
/// returns.
|
||
///
|
||
/// **THREE-valued, and that is the whole point of the function.** `Some(true)` = the running server
|
||
/// confirmed it; `Some(false)` = the oracle dropped it confident-bad; `None` = the oracle could not check
|
||
/// it at all (a library this run did not map, a non-vtable class), and the artifact must say so rather
|
||
/// than pick a side. A two-state reading — "present means good, absent means bad" — is exactly the
|
||
/// collapse this exists to prevent, because it would report an unchecked entry as validated.
|
||
///
|
||
/// Only the shipped tiers (`core` + `high_confidence`) are annotated; `experimental` was never sent to the
|
||
/// server, so it keeps `None`. Keeps the monolith self-contained — no standalone validation sidecar.
|
||
pub(crate) fn annotate_validation(
|
||
mono: &mut model::Monolith,
|
||
verdicts: &BTreeMap<String, Option<bool>>,
|
||
) {
|
||
// `verdicts` carries the HONEST per-entry outcome from the live oracle: Some(true) = live-confirmed,
|
||
// None = kept but the oracle could NOT check it (library not mapped this run, non-vtable class, or an
|
||
// out-of-bounds-but-kept offset that was demoted to Some(false)) — so the artifact must not claim
|
||
// validation; an entry ABSENT from `verdicts` was dropped by the oracle -> Some(false). The monolith must
|
||
// never assert a live validation it did not receive ("never lies").
|
||
let annotate = |m: &mut BTreeMap<String, model::MonoEntry>| {
|
||
for (name, e) in m.iter_mut() {
|
||
e.validated = verdicts.get(name).copied().unwrap_or(Some(false));
|
||
}
|
||
};
|
||
annotate(&mut mono.core);
|
||
annotate(&mut mono.high_confidence);
|
||
let all = || mono.core.values().chain(mono.high_confidence.values());
|
||
let validated = all().filter(|e| e.validated == Some(true)).count();
|
||
let unverified = all().filter(|e| e.validated.is_none()).count();
|
||
let failed = all().filter(|e| e.validated == Some(false)).count();
|
||
eprintln!(
|
||
" monolith validation annotated: {validated} live-validated, {unverified} kept-unverified, \
|
||
{failed} failed (core + high_confidence)"
|
||
);
|
||
}
|
||
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// §6 · CORPUS-MODEL DISTILLATION
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
|
||
/// A vtable-offset function's class + its dated offset history (from gamedata across eras).
|
||
struct VtFunc {
|
||
name: String,
|
||
class: String,
|
||
timeline: Vec<(String, i64)>, // (date, offset), sorted ascending by date
|
||
}
|
||
|
||
fn parse_offset(v: &str) -> Option<i64> {
|
||
v.rsplit('=').next().unwrap_or(v).trim().parse().ok() // handles "102" and "linux=102"
|
||
}
|
||
|
||
/// Build each catalogue function's dated vtable-offset timeline: sorted `(YYYY-MM-DD, offset)` pairs,
|
||
/// skipping functions with no dated linux/any vtable-offset variant. The shared input to both the model
|
||
/// distiller's slot chaining and gamedata's offset derivation — each offset variant's `src` carries its
|
||
/// own date (baked into the catalogue), so there is no external tag→date map.
|
||
fn vtable_offset_timelines(cat: &[Func]) -> Vec<VtFunc> {
|
||
let mut vfuncs: Vec<VtFunc> = Vec::new();
|
||
for f in cat {
|
||
let mut tl: Vec<(String, i64)> = Vec::new();
|
||
for v in &f.variants {
|
||
if v.kind != VariantKind::VtableOffset || !v.platform.is_linux() {
|
||
continue;
|
||
}
|
||
// `src` is the offset's `YYYY-MM-DD` commit/build date.
|
||
let d = (v.src.len() >= 10 && v.src.as_bytes().get(4) == Some(&b'-'))
|
||
.then(|| v.src.clone());
|
||
if let (Some(d), Some(o)) = (d, parse_offset(&v.value)) {
|
||
tl.push((d, o));
|
||
}
|
||
}
|
||
if tl.is_empty() {
|
||
continue;
|
||
}
|
||
tl.sort();
|
||
vfuncs.push(VtFunc {
|
||
name: f.name.clone(),
|
||
class: class_of(&f.name).to_string(),
|
||
timeline: tl,
|
||
});
|
||
}
|
||
vfuncs
|
||
}
|
||
|
||
/// Fingerprint every slot address of a vtable: `slot index -> its structural fingerprint` (`None` where
|
||
/// a slot doesn't decode). The per-class vtable signature the cross-build slot alignment operates on.
|
||
fn slot_fingerprints(img: &CodeImage, slots: &[u64]) -> Vec<Option<Vec<u32>>> {
|
||
slots
|
||
.iter()
|
||
.map(|&a| fingerprint::extract(img, a).map(|f| f.to_vec()))
|
||
.collect()
|
||
}
|
||
|
||
/// As [`slot_fingerprints`], but MEMOIZES `fingerprint::extract` per function address — a single inherited
|
||
/// method (a `CBaseEntity` slot) appears at the same address in hundreds of subclasses' vtables, so the
|
||
/// distill re-fingerprints it once per owning class without this. `extract` is pure/deterministic, so the
|
||
/// memo returns byte-identical vectors. CAVEAT: the memo MUST be scoped to ONE image — the offline
|
||
/// server/engine2 `CodeImage`s share low vaddr ranges, so a cross-image bare-vaddr memo would be a
|
||
/// correctness/output bug. `extract_build_vtables` creates a fresh memo per image for exactly this reason.
|
||
fn slot_fingerprints_memo(
|
||
img: &CodeImage,
|
||
slots: &[u64],
|
||
memo: &mut HashMap<u64, Option<Vec<u32>>>,
|
||
) -> Vec<Option<Vec<u32>>> {
|
||
slots
|
||
.iter()
|
||
.map(|&a| {
|
||
memo.entry(a)
|
||
.or_insert_with(|| fingerprint::extract(img, a).map(|f| f.to_vec()))
|
||
.clone()
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
pub(crate) fn build_date(label: &str) -> String {
|
||
label.chars().take(10).collect() // "2024-08-21_231850" -> "2024-08-21"
|
||
}
|
||
|
||
/// The offset in effect at `date` = the latest era observation with date <= `date`.
|
||
fn offset_at(timeline: &[(String, i64)], date: &str) -> Option<i64> {
|
||
timeline
|
||
.iter()
|
||
.rfind(|(d, _)| d.as_str() <= date)
|
||
.map(|(_, o)| *o)
|
||
}
|
||
|
||
/// One build's identity in the corpus model: its directory label and derived date.
|
||
#[derive(serde::Serialize, serde::Deserialize)]
|
||
struct BuildMeta {
|
||
label: String,
|
||
date: String,
|
||
}
|
||
|
||
/// A function's distilled ABI shape — the argument footprint + return class, serializable for the model.
|
||
/// The consensus of the corpus's newest builds; a target whose derived shape differs is a prototype
|
||
/// drift the byte-signature can't see, flaggable at DERIVE time without the raw corpus.
|
||
#[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||
struct AbiSig {
|
||
int: u8,
|
||
float: u8,
|
||
ret: abi::RetClass, // typed; serializes to the describe() token ("ret=int", …)
|
||
}
|
||
|
||
impl From<abi::AbiShape> for AbiSig {
|
||
fn from(s: abi::AbiShape) -> Self {
|
||
AbiSig {
|
||
int: s.int_args,
|
||
float: s.float_args,
|
||
ret: s.ret_class,
|
||
}
|
||
}
|
||
}
|
||
|
||
impl AbiSig {
|
||
/// True when this shape DEFINITELY differs from `other`: the argument footprint moved, or both
|
||
/// return classes are definite (not `Unknown`) and differ. An `Unknown` return is "no signal" and
|
||
/// never raises a drift flag on its own.
|
||
fn differs(&self, other: &AbiSig) -> bool {
|
||
self.int != other.int
|
||
|| self.float != other.float
|
||
|| (self.ret != other.ret
|
||
&& self.ret != abi::RetClass::Unknown
|
||
&& other.ret != abi::RetClass::Unknown)
|
||
}
|
||
fn brief(&self) -> String {
|
||
format!(
|
||
"int={} float={} {}",
|
||
self.int,
|
||
self.float,
|
||
self.ret.describe()
|
||
)
|
||
}
|
||
}
|
||
|
||
/// One reference build's per-sig-fn observations: `(name, fingerprint, optional ABI shape)`.
|
||
type RefObs = Vec<(String, Vec<u32>, Option<AbiSig>)>;
|
||
|
||
/// The model's `consensus_abi` field from its raw `abi_obs` windows — the modal ABI shape per sig fn.
|
||
/// SHARED by the full distill and the incremental fold (like `resolved_slot_of`/`extract_build_vtables`):
|
||
/// `consensus_abi` is a `CorpusModel` field and the fold contracts byte-equality with a re-distill, so an
|
||
/// inline copy edited in one path but not the other would break that silently.
|
||
fn consensus_from_windows(
|
||
abi_obs: &BTreeMap<String, Vec<(u32, AbiSig)>>,
|
||
) -> BTreeMap<String, AbiSig> {
|
||
abi_obs
|
||
.iter()
|
||
.filter_map(|(n, obs)| {
|
||
mode_abi(obs.iter().map(|(_, s)| s.clone()).collect()).map(|s| (n.clone(), s))
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
/// The modal shape across a function's per-build observations — robust to an occasional decode miss.
|
||
/// Ties break to the larger shape by `Ord`, so the consensus is deterministic.
|
||
fn mode_abi(mut obs: Vec<AbiSig>) -> Option<AbiSig> {
|
||
obs.sort();
|
||
let mut best: Option<(AbiSig, usize)> = None;
|
||
let mut i = 0;
|
||
while i < obs.len() {
|
||
let run = 1 + obs[i + 1..].iter().take_while(|x| **x == obs[i]).count();
|
||
if best.as_ref().is_none_or(|(_, c)| run >= *c) {
|
||
best = Some((obs[i].clone(), run));
|
||
}
|
||
i += run;
|
||
}
|
||
best.map(|(s, _)| s)
|
||
}
|
||
|
||
/// A vtable slot index stored in 4 bytes instead of `Option<usize>`'s 16. Slot indices max in the low
|
||
/// hundreds, so a `u32` with `u32::MAX` reserved for "no match" is lossless — and it quarters the RAM of
|
||
/// the model's `hops`/`resolved_slot`, which are its dominant cost on a long-history, many-class corpus
|
||
/// (Dota's clean model is ~10x CS2's in both class-slots and builds; the `Option<usize>` hops alone would
|
||
/// be tens of GB). Serializes to the SAME JSON as `Option<usize>` — a bare number, or `null` for none —
|
||
/// so the model file format and every previously-produced model are unchanged; the win is purely
|
||
/// build-time RAM and on-disk size.
|
||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||
struct Slot(u32);
|
||
|
||
impl Slot {
|
||
const NONE: Slot = Slot(u32::MAX);
|
||
fn from_opt(o: Option<usize>) -> Slot {
|
||
o.map_or(Slot::NONE, |v| Slot(v as u32))
|
||
}
|
||
fn get(self) -> Option<usize> {
|
||
(self.0 != u32::MAX).then_some(self.0 as usize)
|
||
}
|
||
}
|
||
|
||
impl serde::Serialize for Slot {
|
||
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
|
||
match self.get() {
|
||
Some(v) => s.serialize_u64(v as u64),
|
||
None => s.serialize_none(),
|
||
}
|
||
}
|
||
}
|
||
impl<'de> serde::Deserialize<'de> for Slot {
|
||
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Slot, D::Error> {
|
||
Ok(Slot::from_opt(
|
||
Option::<u32>::deserialize(d)?.map(|v| v as usize),
|
||
))
|
||
}
|
||
}
|
||
|
||
/// One consecutive-build vtable alignment (old slot index -> new slot). The ~93 % of alignments that are pure
|
||
/// IDENTITY (slot `i` still maps to slot `i`, no drops) — plus the empty alignment — are carried as just their
|
||
/// LENGTH, both on disk (a single integer, not the full `[0,1,2,…,n-1]` array) AND IN MEMORY (the `Identity`
|
||
/// variant, not a materialized `Vec<Slot>`). `hops` is the model's dominant field at scale (per-class ×
|
||
/// per-build-PAIR, so it grows with corpus length), and it is overwhelmingly identity, so this collapses the
|
||
/// on-disk size, the JSON parse cost, AND the resident RAM + per-load allocations (a model load does not
|
||
/// materialize millions of small identity `Vec`s) — with no derivation change. A shifted alignment keeps the
|
||
/// explicit slot list (`Explicit`, `null` for a dropped slot). `deserialize` accepts BOTH forms, so a full
|
||
/// identity array also loads and normalizes to `Identity`.
|
||
#[derive(Clone, PartialEq, Eq)]
|
||
enum Hop {
|
||
/// identity/empty alignment of this length: slot `i` maps to `i`. The compressible common case.
|
||
Identity(u32),
|
||
/// a shifted alignment: the explicit old-slot -> new-slot list (`Slot::NONE` for a dropped slot).
|
||
Explicit(Vec<Slot>),
|
||
}
|
||
|
||
impl Default for Hop {
|
||
fn default() -> Hop {
|
||
Hop::Identity(0) // the empty alignment
|
||
}
|
||
}
|
||
|
||
impl Hop {
|
||
/// Classify a raw alignment into the compact form: a pure identity/empty run (every present slot maps to
|
||
/// its own index) collapses to `Identity(len)`, anything shifted stays `Explicit`.
|
||
fn from_slots(v: Vec<Slot>) -> Hop {
|
||
if v.iter().enumerate().all(|(i, s)| s.get() == Some(i)) {
|
||
Hop::Identity(v.len() as u32)
|
||
} else {
|
||
Hop::Explicit(v)
|
||
}
|
||
}
|
||
/// The alignment as one `Option<usize>` per old slot — the form the derive/backfill readers consume
|
||
/// (identity materializes `(0..len)` on demand rather than storing it).
|
||
fn to_options(&self) -> Vec<Option<usize>> {
|
||
match self {
|
||
Hop::Identity(n) => (0..*n as usize).map(Some).collect(),
|
||
Hop::Explicit(v) => v.iter().map(|s| s.get()).collect(),
|
||
}
|
||
}
|
||
}
|
||
|
||
impl serde::Serialize for Hop {
|
||
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
|
||
match self {
|
||
Hop::Identity(n) => s.serialize_u64(*n as u64), // identity (incl. empty) -> its length
|
||
Hop::Explicit(v) => v.serialize(s), // shifted -> the explicit slot array
|
||
}
|
||
}
|
||
}
|
||
|
||
impl<'de> serde::Deserialize<'de> for Hop {
|
||
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Hop, D::Error> {
|
||
use serde::de::{self, SeqAccess, Visitor};
|
||
struct HopVisitor;
|
||
impl<'de> Visitor<'de> for HopVisitor {
|
||
type Value = Hop;
|
||
fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||
f.write_str("an integer (identity-alignment length) or an array of slot indices")
|
||
}
|
||
// integer form: an identity alignment of this length -> slot i maps to i.
|
||
fn visit_u64<E: de::Error>(self, v: u64) -> Result<Hop, E> {
|
||
Ok(Hop::Identity(v as u32))
|
||
}
|
||
fn visit_i64<E: de::Error>(self, v: i64) -> Result<Hop, E> {
|
||
self.visit_u64(v as u64)
|
||
}
|
||
// array form: normalize through `from_slots` so a pre-compression identity array collapses to
|
||
// `Identity` (identical to the integer form), a shift stays `Explicit`.
|
||
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Hop, A::Error> {
|
||
let mut v = Vec::new();
|
||
while let Some(s) = seq.next_element::<Slot>()? {
|
||
v.push(s);
|
||
}
|
||
Ok(Hop::from_slots(v))
|
||
}
|
||
}
|
||
d.deserialize_any(HopVisitor)
|
||
}
|
||
}
|
||
|
||
/// Distilled cross-build corpus signals — everything `gamedata` reads from the full build corpus,
|
||
/// WITHOUT the Valve binaries. Fingerprints/offsets/slot-indices are derived facts (zero
|
||
/// Valve bytes) → shippable. Produced by `corpus-model`; consumed by `produce --corpus-model`
|
||
/// alongside only the target build's binary. Incremental: model N + build N+1 → model N+1.
|
||
#[derive(serde::Serialize, serde::Deserialize)]
|
||
pub struct CorpusModel {
|
||
/// The class scope this model was distilled with. The incremental fold recomputes the tracked-class set
|
||
/// with the CLI's `--class-scope`, so a fold whose scope differs from the distill's would silently
|
||
/// produce a model that is NOT the re-distill it claims to be — the fold `ensure!`s they match. Defaults
|
||
/// to `Clean` (what every production model uses) when the field is absent.
|
||
#[serde(default)]
|
||
class_scope: ClassScope,
|
||
builds: Vec<BuildMeta>, // chronological
|
||
/// class -> per consecutive build-pair: the vtable slot alignment (old slot -> new slot). Each alignment
|
||
/// is a [`Hop`], which serializes the identity-alignment common case as a single length integer.
|
||
hops: BTreeMap<String, Vec<Hop>>,
|
||
/// class -> per build: vtable slot count (offset in-bounds checks).
|
||
slot_counts: BTreeMap<String, Vec<usize>>,
|
||
/// sig function -> per build: the vtable slot its signature resolved to (virtual-anchor timeline).
|
||
resolved_slot: BTreeMap<String, Vec<Slot>>,
|
||
/// sig function -> `(build index, raw fingerprint)` across the newest `CORPUS_REF_K` BUILDS, newest
|
||
/// first — the sig-hit verification set.
|
||
///
|
||
/// The build index is what makes the window well-defined. Without it the incremental fold could only
|
||
/// keep "the newest K OBSERVATIONS", which for a function that fails to resolve in some builds retains
|
||
/// fingerprints from arbitrarily far back — a superset of what a re-distill produces, drifting further
|
||
/// with every fold, and one the fold==re-distill equivalence claim could never actually hold against.
|
||
/// With it, the fold evicts by build AGE and reproduces the re-distill's window exactly.
|
||
ref_fps: BTreeMap<String, Vec<(u32, Vec<u32>)>>,
|
||
/// latest build's per-class vtable slot fingerprints — to align the NEXT build without its binary.
|
||
/// (BTreeMap like the other fields, so the serialized model is deterministic; the transient per-build
|
||
/// `VtableFps` stays a HashMap — this is the one boundary where they convert.)
|
||
latest_vtable_fps: BTreeMap<String, Vec<Option<Vec<u32>>>>,
|
||
/// sig function -> its consensus ABI shape across the newest builds. Lets `produce --corpus-model`
|
||
/// flag a target whose derived shape differs from history at DERIVE time — the prototype-drift signal
|
||
/// `abi-diff` gives, but forward and corpus-free.
|
||
consensus_abi: BTreeMap<String, AbiSig>,
|
||
/// sig function -> `(build index, raw ABI shape)` over the newest builds, mirroring [`Self::ref_fps`].
|
||
/// The window `consensus_abi` is the mode of — stored so an incremental fold can re-window it and
|
||
/// recompute the consensus byte-exactly rather than approximate from the collapsed mode.
|
||
abi_obs: BTreeMap<String, Vec<(u32, AbiSig)>>,
|
||
}
|
||
|
||
/// Number of newest builds whose raw sig fingerprints go into the model's verification set.
|
||
const CORPUS_REF_K: usize = 8;
|
||
|
||
/// The per-build array shape every downstream reader assumes.
|
||
///
|
||
/// `slot_counts`, `hops` and `resolved_slot` are read by build INDEX, so a length desync does not fail —
|
||
/// it silently re-dates an observation, attributing this build's slot to some older one. Checked where the
|
||
/// model is WRITTEN rather than only where it is rolled forward, because a distill that emits a malformed
|
||
/// model is the same defect one fold later and the fold's error would then name the wrong producer.
|
||
/// O(classes) against a pass that just walked whole binaries, so it runs on the production path rather
|
||
/// than behind a debug assert.
|
||
fn check_model_shape(m: &CorpusModel) -> Result<()> {
|
||
let nb = m.builds.len();
|
||
for (c, v) in &m.slot_counts {
|
||
ensure!(
|
||
v.len() == nb,
|
||
"slot_counts[{c}] has {} entries, expected {nb}",
|
||
v.len()
|
||
);
|
||
}
|
||
for (c, v) in &m.hops {
|
||
ensure!(
|
||
v.len() == nb - 1,
|
||
"hops[{c}] has {} entries, expected {}",
|
||
v.len(),
|
||
nb - 1
|
||
);
|
||
}
|
||
for (n, v) in &m.resolved_slot {
|
||
ensure!(
|
||
v.len() == nb,
|
||
"resolved_slot[{n}] has {} entries, expected {nb}",
|
||
v.len()
|
||
);
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// The set of classes a model tracks hops for — THE rule that has to agree between the full distill and the
|
||
/// incremental fold, since `fold_model_cmd` claims to produce a model equal to a re-distill over the same
|
||
/// builds and CI depends on that. Shared for the same reason `extract_build_vtables` and `ref_obs_of_build`
|
||
/// are: a divergence here would be silent, and the fold is the standing production path.
|
||
///
|
||
/// `clean` (default) = every REAL game class in the latest build — enough for any modding/PR offset to
|
||
/// derive model-only, without the dead weight (template stamps / protobuf / NetworkVar chainers whose hops
|
||
/// nobody derives an offset from, and whose ML "patterns" live in the fingerprint matrix, not here). `all`
|
||
/// keeps those too; `catalogue` keeps only what the catalogue names. The catalogue's own classes are ALWAYS
|
||
/// unioned in (the derivation needs them). CI compresses the model on release, so the uncompressed on-disk
|
||
/// size is not the shipped size.
|
||
fn tracked_classes(
|
||
prof: &GameProfile,
|
||
class_scope: ClassScope,
|
||
latest_imgs: &[CodeImage],
|
||
vfuncs: &[VtFunc],
|
||
sig_funcs: &[&Func],
|
||
) -> Vec<String> {
|
||
let class_ok = |name: &str| match class_scope {
|
||
ClassScope::All => true,
|
||
ClassScope::Catalogue => false,
|
||
ClassScope::Clean => {
|
||
!is_dead_weight_class(prof, name)
|
||
&& !name.contains('<')
|
||
&& !name.contains("NetworkVar_")
|
||
}
|
||
};
|
||
latest_imgs
|
||
.iter()
|
||
.flat_map(|img| rtti::enumerate_vtables(img, prof.max_vtable_slots))
|
||
.filter(|c| c.offset_to_top == 0)
|
||
.map(|c| c.name)
|
||
.filter(|n| class_ok(n))
|
||
.chain(vfuncs.iter().map(|f| f.class.clone()))
|
||
.chain(sig_funcs.iter().map(|f| class_of(&f.name).to_string()))
|
||
.collect::<BTreeSet<_>>()
|
||
.into_iter()
|
||
.collect()
|
||
}
|
||
|
||
pub fn corpus_model_cmd(
|
||
prof: &GameProfile,
|
||
catalogue: &Path,
|
||
corpus: &Path,
|
||
class_scope: ClassScope,
|
||
out: &Path,
|
||
) -> Result<()> {
|
||
let cat = load_catalogue(catalogue)?;
|
||
|
||
// dated vtable-offset timelines -> the classes we chain (same as gamedata).
|
||
let vfuncs = vtable_offset_timelines(&cat);
|
||
let sig_funcs: Vec<&Func> = cat.iter().filter(|f| !linux_sigs(f).is_empty()).collect();
|
||
|
||
let mut builds: Vec<(String, PathBuf)> = find_builds(prof, corpus)?
|
||
.into_iter()
|
||
.map(|p| (build_date(&label_of(&p)), p))
|
||
.collect();
|
||
builds.sort();
|
||
ensure!(!builds.is_empty(), "no builds under {}", corpus.display());
|
||
|
||
// SCOPED so the newest build's images are freed before the streaming loop starts. Holding them for
|
||
// the whole run would pin one extra complete image set — file bytes plus each image's relocation and
|
||
// symbol maps — against this command's whole point, which is bounded RAM over ~1k builds.
|
||
let classes = {
|
||
let latest_imgs = load_build_images(prof, &builds.last().unwrap().1);
|
||
tracked_classes(prof, class_scope, &latest_imgs, &vfuncs, &sig_funcs)
|
||
};
|
||
eprintln!(
|
||
"distilling {} builds x {} classes (scope={class_scope:?}; reads the corpus once) ...",
|
||
builds.len(),
|
||
classes.len()
|
||
);
|
||
|
||
// The one expensive pass over the binaries: each class's vtable slot fingerprints+addresses and each sig
|
||
// function's resolved address, per build. Builds are processed ONE AT A TIME in chronological order — the
|
||
// parallelism lives INSIDE each build's extract (the `resolve_unique` sweep is ~98% of the cost and fans
|
||
// out across all cores; see `extract_build_vtables`), so a single build saturates the machine while only
|
||
// that build's fps (plus the previous build's, for the next hop) are ever resident. That keeps peak RAM at
|
||
// ~1.5 builds — which is what makes a deep game (Dota's fat fps)
|
||
// fit. The fat per-slot fingerprints are the memory hog; the distilled signals below consume them only
|
||
// pairwise (nw_align of consecutive builds) plus the final build's set, so we carry only the previous
|
||
// build's fps across the loop and drop the rest. Byte-identical to a whole-corpus distill (same
|
||
// consecutive pairs, same order).
|
||
let want: HashSet<&str> = classes.iter().map(String::as_str).collect();
|
||
let nb = builds.len();
|
||
let mut hop_acc: HashMap<String, Vec<Vec<Slot>>> = classes
|
||
.iter()
|
||
.map(|c| (c.clone(), Vec::with_capacity(nb.saturating_sub(1))))
|
||
.collect();
|
||
let mut count_acc: HashMap<String, Vec<usize>> = classes
|
||
.iter()
|
||
.map(|c| (c.clone(), Vec::with_capacity(nb)))
|
||
.collect();
|
||
let mut rslot_acc: HashMap<String, Vec<Slot>> = sig_funcs
|
||
.iter()
|
||
.map(|f| (f.name.clone(), Vec::with_capacity(nb)))
|
||
.collect();
|
||
let mut prev_fps: Option<VtableFps> = None;
|
||
let nthreads = default_threads(None);
|
||
for (_, dir) in &builds {
|
||
let bv = extract_build_vtables(prof, dir, &want, &sig_funcs, nthreads);
|
||
// A build whose libs are present-but-unparseable poisons every chain that crosses it (empty
|
||
// fingerprints ⇒ empty hops ⇒ an unconditional chain break), and `find_builds` admits builds on
|
||
// file existence alone — so say so instead of folding the damage in silently.
|
||
if bv.empty {
|
||
bail!(
|
||
"build {} loaded ZERO libraries — it would contribute empty hops and sever every \
|
||
anchor chain crossing it; remove or repair it, then re-distill",
|
||
bv.date
|
||
);
|
||
}
|
||
if !bv.unreadable.is_empty() {
|
||
eprintln!(
|
||
" WARNING build {}: {} unparseable ({}) — its fingerprints are partial",
|
||
bv.date,
|
||
bv.unreadable.len(),
|
||
bv.unreadable.join(", ")
|
||
);
|
||
}
|
||
// Per-class facts — the slot count and the vtable-slot hop against the previous build — computed in
|
||
// PARALLEL over the class set. `parallel_map` returns results in input (class) order, so pushing them
|
||
// into the by-name accumulators below is byte-identical to a serial per-class loop —
|
||
// the build loop itself stays serial (chronology is the hop's `from`, and each key still gets one push
|
||
// per build). count is folded into the same pass (trivial, but it spares a second walk of the classes).
|
||
let per_class: Vec<(usize, Option<Vec<Slot>>)> = match &prev_fps {
|
||
Some(prev) => parallel_map(&classes, nthreads, |c| {
|
||
let count = bv.fps.get(c).map_or(0, Vec::len);
|
||
// hop empty where either side lacks the class — identical to the windows(2) pairing of a
|
||
// whole-corpus distill.
|
||
let hv: Vec<Slot> = match (prev.get(c), bv.fps.get(c)) {
|
||
(Some(a), Some(b)) => nw_align(a, b).into_iter().map(Slot::from_opt).collect(),
|
||
_ => Vec::new(),
|
||
};
|
||
(count, Some(hv))
|
||
}),
|
||
// first build: a count for every class, but no hop yet (nothing to align against).
|
||
None => parallel_map(&classes, nthreads, |c| {
|
||
(bv.fps.get(c).map_or(0, Vec::len), None)
|
||
}),
|
||
};
|
||
for (c, (count, hop)) in classes.iter().zip(per_class) {
|
||
count_acc.get_mut(c).unwrap().push(count);
|
||
if let Some(hv) = hop {
|
||
hop_acc.get_mut(c).unwrap().push(hv);
|
||
}
|
||
}
|
||
// the vtable slot each sig fn resolved to this build (position of its resolved addr in the vtable),
|
||
// likewise parallel over the sig set and reassembled in input order.
|
||
let rslots: Vec<Slot> = parallel_map(&sig_funcs, nthreads, |f| {
|
||
Slot::from_opt(resolved_slot_of(&bv, f))
|
||
});
|
||
for (f, s) in sig_funcs.iter().zip(rslots) {
|
||
rslot_acc.get_mut(&f.name).unwrap().push(s);
|
||
}
|
||
prev_fps = Some(bv.fps); // carry only this build's fps: next hop's `from`, and at the end `latest`
|
||
}
|
||
|
||
// distill -> the small, shippable signals (assembled in classes / sig_funcs key order)
|
||
let hops: BTreeMap<String, Vec<Hop>> = classes
|
||
.iter()
|
||
.map(|c| {
|
||
(
|
||
c.clone(),
|
||
hop_acc
|
||
.remove(c)
|
||
.unwrap_or_default()
|
||
.into_iter()
|
||
.map(Hop::from_slots)
|
||
.collect(),
|
||
)
|
||
})
|
||
.collect();
|
||
let slot_counts: BTreeMap<String, Vec<usize>> = classes
|
||
.iter()
|
||
.map(|c| (c.clone(), count_acc.remove(c).unwrap_or_default()))
|
||
.collect();
|
||
let resolved_slot: BTreeMap<String, Vec<Slot>> = sig_funcs
|
||
.iter()
|
||
.map(|f| {
|
||
(
|
||
f.name.clone(),
|
||
rslot_acc.remove(&f.name).unwrap_or_default(),
|
||
)
|
||
})
|
||
.collect();
|
||
// convert the transient (HashMap) newest-build fps into the model's deterministic BTreeMap field.
|
||
let latest_vtable_fps: BTreeMap<String, Vec<Option<Vec<u32>>>> =
|
||
prev_fps.unwrap_or_default().into_iter().collect();
|
||
|
||
// reference fingerprints of each sig fn in the newest CORPUS_REF_K builds (verification set, raw)
|
||
// newest-first, carrying each build's ABSOLUTE index so the window is defined by build age (see
|
||
// `CorpusModel::ref_fps`) rather than by how many observations happened to succeed.
|
||
let ref_dirs: Vec<(u32, PathBuf)> = builds
|
||
.iter()
|
||
.enumerate()
|
||
.rev()
|
||
.take(CORPUS_REF_K)
|
||
.map(|(i, (_, p))| (i as u32, p.clone()))
|
||
.collect();
|
||
// SERIAL over the (few) ref builds so `ref_obs_of_build`'s inner sig fan-out (nthreads-wide) drives the
|
||
// cores directly — an outer parallel_map here would nest and oversubscribe. Order is preserved either way
|
||
// (results keyed by build index + name downstream), so this is byte-identical.
|
||
let per_ref: Vec<(u32, RefObs)> = ref_dirs
|
||
.iter()
|
||
.map(|(i, dir)| (*i, ref_obs_of_build(prof, dir, &cat, &sig_funcs, nthreads)))
|
||
.collect();
|
||
let mut ref_fps: BTreeMap<String, Vec<(u32, Vec<u32>)>> = BTreeMap::new();
|
||
let mut abi_obs: BTreeMap<String, Vec<(u32, AbiSig)>> = BTreeMap::new();
|
||
for (idx, v) in per_ref {
|
||
for (n, fp, sh) in v {
|
||
if let Some(s) = sh {
|
||
abi_obs.entry(n.clone()).or_default().push((idx, s));
|
||
}
|
||
ref_fps.entry(n).or_default().push((idx, fp));
|
||
}
|
||
}
|
||
// consensus ABI shape per sig fn = the modal shape over the newest builds (derive-time drift check).
|
||
// Computed by reference so the raw `abi_obs` window survives into the model (the incremental fold needs it).
|
||
let consensus_abi = consensus_from_windows(&abi_obs);
|
||
|
||
let model = CorpusModel {
|
||
class_scope,
|
||
builds: builds
|
||
.iter()
|
||
.map(|(d, p)| BuildMeta {
|
||
label: label_of(p),
|
||
date: d.clone(),
|
||
})
|
||
.collect(),
|
||
hops,
|
||
slot_counts,
|
||
resolved_slot,
|
||
ref_fps,
|
||
latest_vtable_fps,
|
||
consensus_abi,
|
||
abi_obs,
|
||
};
|
||
// Checked HERE too, not only in the fold: a distill that emits a desynced model is the same defect
|
||
// one build later, and the fold's error would then name the wrong producer.
|
||
check_model_shape(&model)?;
|
||
let text = serde_json::to_string(&model)?;
|
||
std::fs::write(out, &text).with_context(|| format!("write {}", out.display()))?;
|
||
eprintln!(
|
||
"corpus model: {} builds, {} classes, {} sig fns, {:.1} MB -> {}",
|
||
model.builds.len(),
|
||
classes.len(),
|
||
sig_funcs.len(),
|
||
text.len() as f64 / 1e6,
|
||
out.display()
|
||
);
|
||
Ok(())
|
||
}
|
||
|
||
/// One build's tracked-class vtable fingerprints/addresses plus each sig fn's uniquely-resolved address —
|
||
/// the per-build extraction shared by the full distill (`corpus-model`) and the incremental fold. One RTTI
|
||
/// sweep per image (imgs are [server, engine2, …] — the first image to carry a class wins), keeping only the
|
||
/// `want` classes' primary vtables: O(sweep + classes), not O(classes × sweep).
|
||
fn extract_build_vtables(
|
||
prof: &GameProfile,
|
||
dir: &Path,
|
||
want: &HashSet<&str>,
|
||
sig_funcs: &[&Func],
|
||
nthreads: usize,
|
||
) -> BuildVtables {
|
||
let (imgs, unreadable) = load_build_images_counted(prof, dir);
|
||
let (mut fps, mut addrs) = (HashMap::new(), HashMap::new());
|
||
let mut truncated: Vec<String> = Vec::new();
|
||
for img in &imgs {
|
||
// Fresh memo per IMAGE (never shared across images — see slot_fingerprints_memo's caveat): inherited
|
||
// slots repeat across this image's subclass vtables, so this collapses the fingerprint pass.
|
||
let mut fp_memo: HashMap<u64, Option<Vec<u32>>> = HashMap::new();
|
||
for cv in rtti::enumerate_vtables(img, prof.max_vtable_slots)
|
||
.into_iter()
|
||
.filter(|c| c.offset_to_top == 0)
|
||
{
|
||
if want.contains(cv.name.as_str()) && !fps.contains_key(&cv.name) {
|
||
// Exactly at the cap means the read STOPPED there, not that the vtable ended there — the
|
||
// tail (and any offset in it) is missing from the model with nothing else to show for it.
|
||
if cv.slots.len() == prof.max_vtable_slots {
|
||
truncated.push(cv.name.clone());
|
||
}
|
||
fps.insert(
|
||
cv.name.clone(),
|
||
slot_fingerprints_memo(img, &cv.slots, &mut fp_memo),
|
||
);
|
||
addrs.insert(cv.name.clone(), cv.slots);
|
||
}
|
||
}
|
||
}
|
||
if !truncated.is_empty() {
|
||
truncated.sort();
|
||
eprintln!(
|
||
" WARNING {}: {} class(es) hit the {}-slot vtable cap (truncated — raise \
|
||
GameProfile::max_vtable_slots for {}): {}{}",
|
||
label_of(dir),
|
||
truncated.len(),
|
||
prof.max_vtable_slots,
|
||
prof.token,
|
||
truncated
|
||
.iter()
|
||
.take(5)
|
||
.cloned()
|
||
.collect::<Vec<_>>()
|
||
.join(", "),
|
||
if truncated.len() > 5 { ", …" } else { "" }
|
||
);
|
||
}
|
||
// `resolve_unique` dominates the whole extract — a `scan_sig_hits` sweep of every sig fn across all the
|
||
// build's images (the RTTI+fingerprint pass above is a tiny fraction of it). It holds no per-build
|
||
// fingerprints, so parallelising it over the sig set is RSS-NEUTRAL, and the result is keyed by name so it
|
||
// is byte-identical to a serial resolve. This is the distill's real hot loop (see the
|
||
// build-serial driver in `corpus_model_cmd`, which lets this saturate all cores at ~1.5 builds resident).
|
||
let resolved: HashMap<String, u64> = parallel_map(sig_funcs, nthreads, |f| {
|
||
resolve_unique(f, &imgs).map(|a| (f.name.clone(), a))
|
||
})
|
||
.into_iter()
|
||
.flatten()
|
||
.collect();
|
||
BuildVtables {
|
||
date: build_date(&label_of(dir)),
|
||
fps,
|
||
addrs,
|
||
resolved,
|
||
unreadable,
|
||
empty: imgs.is_empty(),
|
||
}
|
||
}
|
||
|
||
/// Unique address of `f` scanning a build's loaded images directly (server+engine2 as a set).
|
||
fn resolve_unique(f: &Func, imgs: &[CodeImage]) -> Option<u64> {
|
||
// Union the per-image unique-match addresses; resolve iff exactly one distinct address wins across all
|
||
// images (vote counts are irrelevant here — only the distinct-address set matters).
|
||
let mut keys: BTreeSet<u64> = BTreeSet::new();
|
||
for img in imgs {
|
||
keys.extend(scan_sig_hits(f, img).into_keys());
|
||
}
|
||
(keys.len() == 1).then(|| *keys.iter().next().unwrap())
|
||
}
|
||
|
||
/// The vtable slot a sig fn's resolved address lands in for one build — the fourth per-build "fact" the
|
||
/// model records (beside slot_counts / hops / ref windows). SHARED by the full distill and the incremental
|
||
/// fold for the same reason its three siblings (`extract_build_vtables`/`ref_obs_of_build`/`tracked_classes`)
|
||
/// are: the fold claims byte-equality with a re-distill, and an inline copy edited in one path but not the
|
||
/// other would break that silently.
|
||
fn resolved_slot_of(bv: &BuildVtables, f: &Func) -> Option<usize> {
|
||
let class = class_of(&f.name);
|
||
bv.resolved.get(&f.name).and_then(|addr| {
|
||
bv.addrs
|
||
.get(class)
|
||
.and_then(|a| a.iter().position(|x| x == addr))
|
||
})
|
||
}
|
||
|
||
/// One reference build's per-sig-fn `(name, fingerprint, optional ABI shape)` observations — the ref-window
|
||
/// extraction shared by the full distill and the incremental fold. Uses `locate_addr` (the same path the
|
||
/// live-derive verification takes), not the chunk loop's `resolve_unique`. The per-sig work
|
||
/// (locate_addr + fingerprint + abi_shape, all thread-safe reads over `rimg`) is parallelised `nthreads`-wide:
|
||
/// in the FOLD this is the sole per-sig pass (a serial 1-wide stretch otherwise), and in the distill the
|
||
/// caller drives the ref builds SERIALLY so this inner fan-out uses the cores rather than nesting under an
|
||
/// outer parallel_map. `parallel_map` preserves input order, so the output equals the old serial `filter_map`.
|
||
fn ref_obs_of_build(
|
||
prof: &GameProfile,
|
||
dir: &Path,
|
||
cat: &[Func],
|
||
sig_funcs: &[&Func],
|
||
nthreads: usize,
|
||
) -> RefObs {
|
||
let (rimg, _unreadable) = preload_images(prof, cat, dir);
|
||
// The identity check belongs HERE, not only where the artifact is emitted. This function is what
|
||
// teaches the model what a name looks like, and it is shared by the distill and the incremental fold —
|
||
// so a contradiction rejected only at emit time would leave the model still learning the wrong
|
||
// function's fingerprint, and the strict fingerprint check would then CONFIRM that wrong resolution on
|
||
// the next build. Measured: that is exactly how one bad entry survived three guards and shipped.
|
||
let ident = Identity::of(&rimg);
|
||
parallel_map(sig_funcs, nthreads, |f| {
|
||
let (img, addr, _) = locate_addr_ident(prof, f, &rimg, Some(&ident))?;
|
||
let fp = fingerprint::extract(img, addr)?.to_vec();
|
||
Some((
|
||
f.name.clone(),
|
||
fp,
|
||
abi::abi_shape(img, addr).map(Into::into),
|
||
))
|
||
})
|
||
.into_iter()
|
||
.flatten()
|
||
.collect()
|
||
}
|
||
|
||
/// Parse a distilled corpus model from disk. Shared by the standalone `fold-model` command and `produce`
|
||
/// (which parses it ONCE, lets the derive borrow it, then hands it by value to the sidecar fold).
|
||
pub fn load_model(path: &Path) -> Result<CorpusModel> {
|
||
let text = std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
|
||
serde_json::from_str(&text).with_context(|| format!("parse {}", path.display()))
|
||
}
|
||
|
||
/// Fold ONE new build into an existing corpus model: `model N + build N+1 → model N+1`, reading only the
|
||
/// loaded model + the one new binary (no corpus). The model is DESIGNED for this — `latest_vtable_fps` holds
|
||
/// the previous build's per-class fingerprints, exactly the seed for the new hop.
|
||
///
|
||
/// EQUIVALENCE — the result equals a full re-distill over the same N+1 builds ONLY when the fold is run with
|
||
/// the same `class_scope` (asserted below) AND the same catalogue the model was distilled from, AND no class
|
||
/// re-appears across the model's latest-build boundary (present in old builds, absent from the model's latest,
|
||
/// present again now). In those cases a class/sig-fn the model has never seen is back-filled with ABSENT
|
||
/// history (a re-distill would instead resolve its real slots/counts/hops in the old binaries) — reduced
|
||
/// coverage for that one name until the next full re-distill, never a wrong offset. The catalogue is not
|
||
/// fingerprint-checked (production always folds with the distill's catalogue), but a newcomer is WARNED below.
|
||
pub fn fold_model_cmd(
|
||
prof: &GameProfile,
|
||
mut model: CorpusModel,
|
||
catalogue: &Path,
|
||
build: &Path,
|
||
class_scope: ClassScope,
|
||
out: &Path,
|
||
) -> Result<()> {
|
||
let cat = load_catalogue(catalogue)?;
|
||
let vfuncs = vtable_offset_timelines(&cat);
|
||
let sig_funcs: Vec<&Func> = cat.iter().filter(|f| !linux_sigs(f).is_empty()).collect();
|
||
|
||
ensure!(
|
||
model.class_scope == class_scope,
|
||
"the input model was distilled with class-scope {:?}, but the fold was called with {:?} — they must \
|
||
match or the folded model would diverge from a re-distill; re-run with --class-scope {:?}",
|
||
model.class_scope,
|
||
class_scope,
|
||
model.class_scope
|
||
);
|
||
|
||
// The new build becomes the latest, so its RTTI (scope-filtered) ∪ catalogue classes IS the new tracked
|
||
// set — exactly as a full distill takes `classes` from ITS latest build (hence the SHARED
|
||
// `tracked_classes`). So a class dropped from the new build's RTTI (and not catalogue-required) falls
|
||
// out; a class new to it is back-filled below.
|
||
// SCOPED, for the reason the move-not-clone below exists: `extract_build_vtables` loads this same
|
||
// build again, so leaving these alive would peak at TWO complete image sets and then hold the dead
|
||
// one through the model rebuild and its multi-hundred-MB serialization.
|
||
let classes = {
|
||
let latest_imgs = load_build_images(prof, build);
|
||
tracked_classes(prof, class_scope, &latest_imgs, &vfuncs, &sig_funcs)
|
||
};
|
||
|
||
let want: HashSet<&str> = classes.iter().map(String::as_str).collect();
|
||
let bv = extract_build_vtables(prof, build, &want, &sig_funcs, default_threads(None));
|
||
// Same guard the full distill enforces: a build that loaded zero libraries contributes empty
|
||
// fingerprints → empty hops → an unconditional chain break severing every anchor older than it. The
|
||
// fold is the standing production path (the sidecar in every `produce`) and its output SEEDS the next
|
||
// build's derive, so folding damage in silently is worse here than in the distill, not better.
|
||
ensure!(
|
||
!bv.empty,
|
||
"build {} loaded ZERO libraries — folding it would sever every anchor chain crossing it; \
|
||
repair or skip this build",
|
||
bv.date
|
||
);
|
||
if !bv.unreadable.is_empty() {
|
||
eprintln!(
|
||
" WARNING build {}: {} unparseable ({}) — its fingerprints are partial",
|
||
bv.date,
|
||
bv.unreadable.len(),
|
||
bv.unreadable.join(", ")
|
||
);
|
||
}
|
||
let old_nb = model.builds.len();
|
||
|
||
// Per class: append the new build's slot count and ONE hop. A continuing class extends its history; a
|
||
// class new to this build is back-filled with absent history (0 count / empty hop for every prior build),
|
||
// matching what a re-distill produces for a class first seen at build N.
|
||
let mut hops: BTreeMap<String, Vec<Hop>> = BTreeMap::new();
|
||
let mut slot_counts: BTreeMap<String, Vec<usize>> = BTreeMap::new();
|
||
let mut backfilled = 0usize; // classes/sig-fns the model has never seen (see the EQUIVALENCE note above)
|
||
for c in &classes {
|
||
let new_count = bv.fps.get(c).map_or(0, Vec::len);
|
||
// MOVE the class's prior counts/hops out of the owned model instead of cloning them (the model is
|
||
// dropped at end of fn and `hops` is its size-dominant field): `remove` returns the exact Vec a
|
||
// clone would have produced, and nothing reads `model.slot_counts`/`model.hops` after this loop, so
|
||
// the folded model is byte-identical. Halves the fold's transient peak RSS.
|
||
match model.slot_counts.remove(c) {
|
||
Some(mut counts) => {
|
||
counts.push(new_count);
|
||
slot_counts.insert(c.clone(), counts);
|
||
// the hop into the new build = align the model's stored latest fps against the new build's
|
||
// (empty where either side lacks the class — identical to the distill's windows(2) pairing).
|
||
let hop: Vec<Slot> = match (model.latest_vtable_fps.get(c), bv.fps.get(c)) {
|
||
(Some(a), Some(b)) => nw_align(a, b).into_iter().map(Slot::from_opt).collect(),
|
||
_ => Vec::new(),
|
||
};
|
||
let mut h = model.hops.remove(c).unwrap_or_default();
|
||
h.push(Hop::from_slots(hop));
|
||
hops.insert(c.clone(), h);
|
||
}
|
||
None => {
|
||
backfilled += 1;
|
||
let mut counts = vec![0usize; old_nb];
|
||
counts.push(new_count);
|
||
slot_counts.insert(c.clone(), counts);
|
||
hops.insert(c.clone(), vec![Hop::default(); old_nb]); // old_nb == new_nb − 1 hops, all empty
|
||
}
|
||
}
|
||
}
|
||
|
||
// Per sig fn: append the vtable slot its resolved address lands in this build (None if unresolved).
|
||
// `resolved_slot` is a per-build parallel array read by build INDEX (`build_per_build_vtables`), so a
|
||
// name absent from the model — a class/sig-fn the CATALOGUE gained since the distill (NOT a contribution:
|
||
// contributions are a derive-time overlay and never enter the model) — must be back-filled with absent
|
||
// history first, exactly as the class loop above. Without it the lone new observation lands at index 0
|
||
// and is attributed to the OLDEST build in the corpus. (Back-fill ≠ a re-distill's real old-build history
|
||
// for such a newcomer — see the EQUIVALENCE note on `fold_model_cmd`; counted + warned below.)
|
||
let mut resolved_slot: BTreeMap<String, Vec<Slot>> = BTreeMap::new();
|
||
for f in &sig_funcs {
|
||
// move-out (see the class loop): the removed Vec == the cloned one; nothing reads the field after.
|
||
let mut v = model.resolved_slot.remove(&f.name).unwrap_or_else(|| {
|
||
backfilled += 1;
|
||
vec![Slot::NONE; old_nb]
|
||
});
|
||
v.push(Slot::from_opt(resolved_slot_of(&bv, f)));
|
||
resolved_slot.insert(f.name.clone(), v);
|
||
}
|
||
|
||
// Ref/ABI windows: prepend the new build's observations (newest-first), then evict everything outside
|
||
// the newest CORPUS_REF_K BUILDS. Evicting by build age — not by list length — is what makes this equal
|
||
// a re-distill: a fn that fails to resolve in some builds would otherwise retain fingerprints from
|
||
// arbitrarily far back, and the divergence would compound over every future fold.
|
||
let new_idx = old_nb as u32; // index of the build being appended = the new latest
|
||
let cutoff = new_idx.saturating_sub(CORPUS_REF_K as u32 - 1); // keep indices >= cutoff
|
||
let mut ref_fps = std::mem::take(&mut model.ref_fps);
|
||
let mut abi_obs = std::mem::take(&mut model.abi_obs);
|
||
for (n, fp, sh) in ref_obs_of_build(prof, build, &cat, &sig_funcs, default_threads(None)) {
|
||
ref_fps
|
||
.entry(n.clone())
|
||
.or_default()
|
||
.insert(0, (new_idx, fp));
|
||
if let Some(s) = sh {
|
||
abi_obs.entry(n).or_default().insert(0, (new_idx, s));
|
||
}
|
||
}
|
||
for v in ref_fps.values_mut() {
|
||
v.retain(|(i, _)| *i >= cutoff);
|
||
}
|
||
for v in abi_obs.values_mut() {
|
||
v.retain(|(i, _)| *i >= cutoff);
|
||
}
|
||
ref_fps.retain(|_, v| !v.is_empty());
|
||
abi_obs.retain(|_, v| !v.is_empty());
|
||
let consensus_abi = consensus_from_windows(&abi_obs);
|
||
|
||
let latest_vtable_fps: BTreeMap<String, Vec<Option<Vec<u32>>>> = bv.fps.into_iter().collect();
|
||
let mut builds = model.builds;
|
||
builds.push(BuildMeta {
|
||
label: label_of(build),
|
||
date: build_date(&label_of(build)),
|
||
});
|
||
|
||
let folded = CorpusModel {
|
||
class_scope,
|
||
builds,
|
||
hops,
|
||
slot_counts,
|
||
resolved_slot,
|
||
ref_fps,
|
||
latest_vtable_fps,
|
||
consensus_abi,
|
||
abi_obs,
|
||
};
|
||
check_model_shape(&folded)?;
|
||
|
||
let out_text = serde_json::to_string(&folded)?;
|
||
std::fs::write(out, &out_text).with_context(|| format!("write {}", out.display()))?;
|
||
eprintln!(
|
||
"folded {} -> model N+1: {} builds, {} classes, {} sig fns, {:.1} MB -> {}",
|
||
label_of(build),
|
||
folded.builds.len(),
|
||
classes.len(),
|
||
sig_funcs.len(),
|
||
out_text.len() as f64 / 1e6,
|
||
out.display()
|
||
);
|
||
if backfilled > 0 {
|
||
eprintln!(
|
||
" NOTE {backfilled} class(es)/sig-fn(s) were new to the model and back-filled with EMPTY \
|
||
pre-fold history — a re-distill would record their real old-build history instead (catalogue \
|
||
grew since the distill). Harmless (reduced coverage for those names only); re-distill for exact."
|
||
);
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// §7 · GAMEDATA DERIVATION (catalogue → signatures / vtable offsets)
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
|
||
/// A per-catalogue-function signature-resolution verdict from [`resolve_signatures`].
|
||
enum SigResolve {
|
||
Skip,
|
||
Flag(String),
|
||
Emit(String, String, String), // confident: (name, library, signature)
|
||
EmitFallback(String, String, String), // heuristic-flagged but uniquely resolves — live-validate
|
||
/// The address resolved, and the BINARY CONTRADICTS the name there — `(name, why)`. Kept distinct from
|
||
/// `Flag` because it is a different fact and a louder one: a drifted sig failed to find its function,
|
||
/// while this one found a function that is provably not it.
|
||
Contradicted(String, String),
|
||
}
|
||
/// What a candidate address is checked AGAINST, in one place because the three travel together: the
|
||
/// cross-build fingerprint history and the bar it has to clear, plus what the binary itself states about
|
||
/// the address ([`Identity`]). Fingerprints answer "is this the same function as last build"; `Identity`
|
||
/// answers "is this the function this name means at all" — different questions, both needed.
|
||
struct Verify<'a> {
|
||
ref_fps: HashMap<String, Vec<Vec<u32>>>,
|
||
ref_majority: usize,
|
||
ident: &'a Identity,
|
||
}
|
||
|
||
/// `(dbg line, resolution, optional (name, abi-drift detail), optional (name, derived ABI shape))` per
|
||
/// catalogue function.
|
||
type SigItem = (
|
||
Option<String>,
|
||
SigResolve,
|
||
Option<(String, String)>,
|
||
Option<(String, model::AbiShape)>,
|
||
);
|
||
|
||
/// The engine's decoded shape as the shipped, format-neutral one. Kept a plain fn rather than a `From` impl
|
||
/// because both types are named `AbiShape` and the conversion crosses the crate boundary in one direction.
|
||
fn shipped_abi(s: abi::AbiShape) -> model::AbiShape {
|
||
model::AbiShape {
|
||
int: s.int_args,
|
||
float: s.float_args,
|
||
stack: s.stack_args,
|
||
ret: s.ret_class.describe().to_string(),
|
||
}
|
||
}
|
||
|
||
/// Resolve every catalogue signature in the target, across threads: locate the era-sig's candidates, verify
|
||
/// the hit against the reference fingerprints (or a self-consensus when history is thin), regenerate a fresh
|
||
/// unique sig, and run the derive-time ABI-drift check. Read-only over `images`/`ref_fps`; the caller applies
|
||
/// the returned verdicts in catalogue order, so `gd`/`sig_ok`/`sig_flag` stay byte-identical to a serial run.
|
||
fn resolve_signatures(
|
||
prof: &GameProfile,
|
||
cat: &[Func],
|
||
images: &HashMap<String, CodeImage>,
|
||
v: &Verify<'_>,
|
||
cmodel: Option<&CorpusModel>,
|
||
dbg: bool,
|
||
) -> Vec<SigItem> {
|
||
let (ref_fps, ref_majority, ident) = (&v.ref_fps, v.ref_majority, v.ident);
|
||
const VERIFY_L1_MAX: u64 = 12;
|
||
parallel_map(cat, default_threads(None), |f| {
|
||
if linux_sigs(f).is_empty() {
|
||
return (None, SigResolve::Skip, None, None);
|
||
}
|
||
let Some((img, lib, cands)) = locate_candidates(prof, f, images) else {
|
||
return (None, SigResolve::Flag(f.name.clone()), None, None);
|
||
};
|
||
let mut dbg_line = None;
|
||
let chosen = match ref_fps.get(&f.name) {
|
||
// stable recent history: pick the target candidate nearest the real function's fingerprint,
|
||
// accept only within recompile distance (rejects a stale sig that drifted onto a decoy).
|
||
Some(rs) if rs.len() >= ref_majority => {
|
||
let best = cands
|
||
.iter()
|
||
.filter_map(|&(addr, _)| {
|
||
let tfp = fingerprint::extract(img, addr)?.to_vec();
|
||
let d = rs.iter().map(|r| l1(&tfp, r)).min().unwrap_or(u64::MAX);
|
||
Some((addr, d))
|
||
})
|
||
.min_by_key(|&(_, d)| d);
|
||
if dbg && let Some((a, d)) = best {
|
||
dbg_line = Some(format!(
|
||
" [dbg] {:<46} @{a:#x} L1={d} ({} refs)",
|
||
f.name,
|
||
rs.len()
|
||
));
|
||
}
|
||
best.filter(|&(_, d)| d <= VERIFY_L1_MAX).map(|(a, _)| a)
|
||
}
|
||
// no stable reference history: trust only a self-corroborating consensus of >=2 distinct era-sigs
|
||
// agreeing in the target itself; a lone hit is otherwise a likely coincidental decoy -> flag.
|
||
_ => {
|
||
if dbg {
|
||
let top = cands.first().map(|&(a, v)| (a, v)).unwrap_or((0, 0));
|
||
dbg_line = Some(format!(
|
||
" [dbg] {:<46} @{:#x} votes={} refs={} -> {}",
|
||
f.name,
|
||
top.0,
|
||
top.1,
|
||
ref_fps.get(&f.name).map_or(0, Vec::len),
|
||
if top.1 >= 2 { "consensus" } else { "FLAG" }
|
||
));
|
||
}
|
||
cands.first().filter(|&&(_, v)| v >= 2).map(|&(a, _)| a)
|
||
}
|
||
};
|
||
// Fallback: the verification heuristic rejected every candidate, but if exactly one era-sig hit
|
||
// uniquely resolves in the target it is a legitimate — if unverified — location; emit it and lean on
|
||
// validate-live as the real gate rather than dropping a real function that merely drifted.
|
||
let was_flagged = chosen.is_none();
|
||
let chosen = chosen.or_else(|| (cands.len() == 1).then(|| cands[0].0));
|
||
// Refuse an address the binary itself says is a different function — including one the fallback
|
||
// above would otherwise emit, since "exactly one era-sig resolves" is precisely how a stale sig's
|
||
// coincidental hit gets through.
|
||
if let Some(addr) = chosen
|
||
&& let Some(why) = ident.contradiction(&lib, img, addr, &f.name)
|
||
{
|
||
return (
|
||
None,
|
||
SigResolve::Contradicted(f.name.clone(), why),
|
||
None,
|
||
None,
|
||
);
|
||
}
|
||
// Derive-time prototype-drift check (model path only): unknown-return-guarded, a kept signal for
|
||
// review, not a drop — the sig still emits.
|
||
// Measured for EVERY resolved signature, not just those with corpus history: it is what lets a
|
||
// declared prototype be machine-checked against this build, and the drift check below is a second
|
||
// reading of the same measurement rather than a separate one.
|
||
let shape = chosen.and_then(|addr| abi::abi_shape(img, addr));
|
||
let abi_drift = shape.and_then(|s| {
|
||
let cons = cmodel?.consensus_abi.get(&f.name)?;
|
||
let sh: AbiSig = s.into();
|
||
sh.differs(cons).then(|| {
|
||
(
|
||
f.name.clone(),
|
||
format!("target [{}] vs history [{}]", sh.brief(), cons.brief()),
|
||
)
|
||
})
|
||
});
|
||
// The measured footprint ships ONLY where the signature does, and the pairing is what makes it
|
||
// meaningful: a `Flag` means this address produced no signature, so the name goes on to
|
||
// `sig_flag` where `recover_by_string_anchor` / `recover_virtual_sigs` may relocate it. `make_sig`
|
||
// is deterministic per (image, address) and already failed here, so any later recovery is
|
||
// provably at a DIFFERENT address — and a footprint measured at this one would then describe some
|
||
// other function while `prototypes::agrees` judged declared types against it. No shape is the
|
||
// acceptable degradation; a shape belonging elsewhere is the thing this project forbids.
|
||
let (r, shape_out) = match chosen
|
||
.and_then(|addr| emit::make_sig(img, addr, CORE_SIG_CAP).map(|s| (lib, s)))
|
||
{
|
||
Some((lib, sig)) if was_flagged => (
|
||
SigResolve::EmitFallback(f.name.clone(), lib, sig),
|
||
shape.map(|s| (f.name.clone(), shipped_abi(s))),
|
||
),
|
||
Some((lib, sig)) => (
|
||
SigResolve::Emit(f.name.clone(), lib, sig),
|
||
shape.map(|s| (f.name.clone(), shipped_abi(s))),
|
||
),
|
||
None => (SigResolve::Flag(f.name.clone()), None),
|
||
};
|
||
(dbg_line, r, abi_drift, shape_out)
|
||
})
|
||
}
|
||
|
||
/// Byte budget for signatures generated by the DERIVE (the `core` tier). Distinct from `--sig-cap`, which
|
||
/// bounds the FOLD's generated sigs: the two run over different inputs, and the derive's value is the one
|
||
/// the shipped guaranteed tier was calibrated against.
|
||
const CORE_SIG_CAP: usize = 400;
|
||
|
||
/// How many of the newest reference builds the sig-hit verification samples when it reads BINARIES; also the
|
||
/// history FLOOR (`/2 + 1` = 3) below which the model path won't trust a fingerprint match. See
|
||
/// [`build_ref_fingerprints`] for why the model path reuses this constant rather than `CORPUS_REF_K`.
|
||
const REF_BUILDS: usize = 5;
|
||
|
||
/// Reference fingerprints for the sig-hit verification, plus the clean-resolve majority threshold: straight
|
||
/// from the corpus MODEL (which ships them), or by resolving + fingerprinting each sig fn in the reference
|
||
/// binaries. (A function resolving cleanly in the majority of sampled builds == a stable recent history;
|
||
/// below that a lone target hit is a suspected decoy.)
|
||
fn build_ref_fingerprints(
|
||
prof: &GameProfile,
|
||
cmodel: Option<&CorpusModel>,
|
||
cat: &[Func],
|
||
ref_dirs: &[PathBuf],
|
||
) -> (HashMap<String, Vec<Vec<u32>>>, usize) {
|
||
if let Some(m) = cmodel {
|
||
// the model ships ref_fps as a (deterministic) BTreeMap; the derive-side verification set is a plain
|
||
// HashMap (order-independent — it's only `.get()`/`.len()`), so copy it across.
|
||
let ref_fps = m
|
||
.ref_fps
|
||
.iter()
|
||
.map(|(k, v)| (k.clone(), v.iter().map(|(_, fp)| fp.clone()).collect()))
|
||
.collect();
|
||
// This threshold is a HISTORY FLOOR, not a per-window majority — deliberately `REF_BUILDS/2+1` (=3),
|
||
// NOT `CORPUS_REF_K/2+1` (=5), even though the model's window is CORPUS_REF_K long. `resolve_signatures`
|
||
// routes on `window.len() >= threshold`: at/above it a function takes the STRICT arm, which accepts the
|
||
// target only within recompile distance of its NEAREST historical fingerprint (`rs.iter().map(l1).min()`
|
||
// — one close ref suffices; it is not a vote). So "resolved cleanly in >=3 of the newest builds" is
|
||
// ample basis for that min-distance check. A higher floor would demote functions with only 3-4 builds
|
||
// of genuine history to the lenient >=2-era-sig arm, where single-candidate ones fall through to the
|
||
// UNVERIFIED fallback — so a higher floor yields MORE unverified entries, not fewer.
|
||
return (ref_fps, REF_BUILDS / 2 + 1);
|
||
}
|
||
let per_ref: Vec<Vec<(String, Vec<u32>)>> =
|
||
parallel_map(ref_dirs, default_threads(None), |dir| {
|
||
let (rimg, _unreadable) = preload_images(prof, cat, dir);
|
||
let mut v = Vec::new();
|
||
for f in cat {
|
||
if linux_sigs(f).is_empty() {
|
||
continue;
|
||
}
|
||
if let Some((img, addr, _)) = locate_addr(prof, f, &rimg)
|
||
&& let Some(fp) = fingerprint::extract(img, addr)
|
||
{
|
||
v.push((f.name.clone(), fp.to_vec()));
|
||
}
|
||
}
|
||
v
|
||
});
|
||
let mut rf: HashMap<String, Vec<Vec<u32>>> = HashMap::new();
|
||
for v in per_ref {
|
||
for (n, fp) in v {
|
||
rf.entry(n).or_default().push(fp);
|
||
}
|
||
}
|
||
let maj = ref_dirs.len() / 2 + 1;
|
||
(rf, maj)
|
||
}
|
||
|
||
/// Apply the (thread-resolved) per-function sig verdicts to `gd` IN CATALOGUE ORDER, so `gd` (name-keyed),
|
||
/// the emitted-count and the flag/fallback lists are byte-identical to a serial resolve. Emits each captured
|
||
/// DBG line and reports the fallback set. Returns `(emitted, flagged, fallback, abi_drift)` — `flagged`/
|
||
/// `emitted` are further mutated by the recovery passes in the caller.
|
||
fn apply_sig_items(
|
||
sig_items: Vec<SigItem>,
|
||
gd: &mut model::Gamedata,
|
||
) -> (
|
||
usize,
|
||
Vec<String>,
|
||
Vec<String>,
|
||
Vec<model::Flagged>,
|
||
BTreeMap<String, model::AbiShape>,
|
||
) {
|
||
let mut sig_ok = 0;
|
||
let mut sig_flag: Vec<String> = Vec::new();
|
||
let mut sig_fallback: Vec<String> = Vec::new();
|
||
let mut contradicted: Vec<model::Flagged> = Vec::new();
|
||
let mut abi_drift: Vec<model::Flagged> = Vec::new();
|
||
let mut shapes: BTreeMap<String, model::AbiShape> = BTreeMap::new();
|
||
for (dbg_line, r, drift, shape) in sig_items {
|
||
if let Some((name, sh)) = shape {
|
||
shapes.insert(name, sh);
|
||
}
|
||
if let Some(l) = dbg_line {
|
||
eprintln!("{l}");
|
||
}
|
||
if let Some((name, detail)) = drift {
|
||
abi_drift.push(model::Flagged::new(
|
||
name,
|
||
model::FlagReason::AbiDrift,
|
||
detail,
|
||
));
|
||
}
|
||
match r {
|
||
SigResolve::Skip => {}
|
||
SigResolve::Flag(name) => sig_flag.push(name),
|
||
SigResolve::Contradicted(name, why) => {
|
||
// Carried with its OWN reason rather than merged into the drifted name list: the artifact
|
||
// must not tell a consumer "no signature resolved" when one did and was refused. It rides
|
||
// the `Flagged` channel (which already carries a reason per entry) rather than widening
|
||
// this function's return.
|
||
contradicted.push(model::Flagged::new(
|
||
name,
|
||
model::FlagReason::NameContradicted,
|
||
why,
|
||
));
|
||
}
|
||
SigResolve::Emit(name, lib, sig) => {
|
||
gd.set_signature(name, lib, sig);
|
||
sig_ok += 1;
|
||
}
|
||
SigResolve::EmitFallback(name, lib, sig) => {
|
||
gd.set_signature(name.clone(), lib, sig);
|
||
sig_ok += 1;
|
||
sig_fallback.push(name);
|
||
}
|
||
}
|
||
}
|
||
if !contradicted.is_empty() {
|
||
eprintln!(
|
||
" {} signatures REFUSED — the binary contradicts the name at the address they resolve to \
|
||
(they ship as `unresolved`, not as a locator):",
|
||
contradicted.len()
|
||
);
|
||
for c in &contradicted {
|
||
eprintln!(" {} — {}", c.name, c.detail);
|
||
}
|
||
}
|
||
abi_drift.extend(contradicted);
|
||
if !sig_fallback.is_empty() {
|
||
eprintln!(
|
||
" {} signatures emitted via unique-resolve fallback (fingerprint-UNVERIFIED — marked \
|
||
`catalogue-unverified` in the monolith): {}",
|
||
sig_fallback.len(),
|
||
sig_fallback.join(", ")
|
||
);
|
||
}
|
||
(sig_ok, sig_flag, sig_fallback, abi_drift, shapes)
|
||
}
|
||
|
||
/// One offline derive, in memory — no files are written; the fold consumes this directly, disk-free.
|
||
pub(crate) struct Derived {
|
||
/// The cssharp-rendered guaranteed gamedata (the `core` tier).
|
||
pub core: String,
|
||
/// Why each catalogue entry failed to produce a shipped locator. Typed, not a JSON string: this crosses
|
||
/// no disk boundary (the fold consumes it in the same call chain), and a typed value makes a renamed
|
||
/// field a compile error rather than a silently absorbed default.
|
||
pub flagged: Vec<model::Flagged>,
|
||
/// Names emitted through the unique-resolve fallback: the reference-fingerprint check REJECTED every
|
||
/// candidate, but exactly one era-sig resolved uniquely, so the locator is sound while the identity is
|
||
/// UNVERIFIED. They land in `core` like any other catalogue entry, so the monolith must mark them —
|
||
/// live validation cannot re-verify them (it checks that a sig resolves to executable code, and the
|
||
/// pattern was generated from that very address, so a decoy passes trivially).
|
||
pub unverified: BTreeSet<String>,
|
||
/// Per-entry argument footprint measured in the target binary — the machine half of the
|
||
/// locator/prototype split, and what a declared prototype is checked against.
|
||
pub abi: BTreeMap<String, model::AbiShape>,
|
||
/// Per-entry string ANCHORS from the catalogue — distinctive literals the function references.
|
||
///
|
||
/// Its own side table for the same reason `abi` is: the derive→fold transport for `core` is the cssharp
|
||
/// locator shape, which has no anchor field, so anything ridden in on an `Entry` there would be dropped
|
||
/// when the fold re-parses it. Carrying them separately keeps the cssharp artifact unchanged — CS# has
|
||
/// no `refs` feature and should not grow a key it cannot read.
|
||
pub anchors: BTreeMap<String, Vec<String>>,
|
||
}
|
||
|
||
/// Derive a target build's gamedata OFFLINE and return it in memory.
|
||
pub(crate) fn gamedata(
|
||
prof: &GameProfile,
|
||
catalogue: &Path,
|
||
source: CorpusSource,
|
||
target: &Path,
|
||
) -> Result<Derived> {
|
||
let emitter = render::by_id("cssharp").expect("cssharp is a built-in format");
|
||
let mut cat = load_catalogue(catalogue)?;
|
||
// Merge contributions BY NAME into the catalogue, not `extend` — a contribution for a name the catalogue
|
||
// already carries must add its dated variant to that Func's `variants` (so `scan_sig_hits`, which votes
|
||
// across one Func's variants, can let it corroborate the existing era-sigs), not append a second Func of
|
||
// the same name that the resolver would treat as an unrelated entry and double-count in the tallies.
|
||
for c in load_contributions(prof, catalogue) {
|
||
match cat.iter_mut().find(|f| f.name == c.name) {
|
||
Some(f) => f.variants.extend(c.variants),
|
||
None => cat.push(c),
|
||
}
|
||
}
|
||
let mut gd = model::Gamedata {
|
||
game_key: prof.game_key.to_string(),
|
||
..Default::default()
|
||
};
|
||
|
||
// dated vtable-offset timelines (consumed by the offset section below).
|
||
let vfuncs = vtable_offset_timelines(&cat);
|
||
|
||
// Corpus MODEL (distilled) OR the raw corpus binaries: with a model, only the target binary is
|
||
// read; the model supplies the reference fingerprints + the cross-build vtable alignment. The model
|
||
// is pre-parsed by the caller (borrowed here, read-only) so `produce` parses it a single time.
|
||
let cmodel: Option<&CorpusModel> = match source {
|
||
CorpusSource::Model(m) => Some(m),
|
||
CorpusSource::Binaries(_) => None,
|
||
};
|
||
// The corpus model is a FORWARD-derivation artifact: it ships cross-build vtable hops plus the
|
||
// fingerprints of its LATEST build only, so the closing alignment hop (model's latest -> target)
|
||
// is reconstructable exactly when the target is newer than every build the model distilled. A
|
||
// target inside the model's range would both lack the fingerprints for its own predecessor hop
|
||
// and let post-target builds pollute the offset chains (the slot would drift the wrong way) — so
|
||
// refuse it up front and point at --corpus rather than emit a quietly-wrong offset. Compare full
|
||
// "YYYY-MM-DD_HHMMSS" labels (they sort chronologically) so a same-day-but-later build still counts.
|
||
if let Some(m) = cmodel
|
||
&& let Some(latest) = m.builds.last()
|
||
{
|
||
let tlabel = label_of(target);
|
||
ensure!(
|
||
tlabel.as_str() > latest.label.as_str(),
|
||
"target build {tlabel} is not newer than the corpus model's latest build {} — \
|
||
--corpus-model derives forward-only. Re-distill a model excluding builds >= the \
|
||
target, or derive this target from --corpus <binaries>.",
|
||
latest.label
|
||
);
|
||
}
|
||
// reference builds strictly before the target's date (binary path only). Empty with a model —
|
||
// which also auto-skips context recovery below (it has no reference binary to build an xref over).
|
||
let ref_builds: Vec<(String, PathBuf)> = match source {
|
||
CorpusSource::Model(_) => Vec::new(),
|
||
CorpusSource::Binaries(corpus) => {
|
||
let tdate = build_date(&label_of(target));
|
||
let mut rb: Vec<(String, PathBuf)> = find_builds(prof, corpus)?
|
||
.into_iter()
|
||
.map(|p| (build_date(&label_of(&p)), p))
|
||
.filter(|(d, _)| d.as_str() < tdate.as_str())
|
||
.collect();
|
||
rb.sort();
|
||
rb
|
||
}
|
||
};
|
||
|
||
// ---- signatures: resolve a known sig in the target, VERIFY the hit is really the function
|
||
// (fingerprint vs the newest reference builds — a stale era-sig can uniquely match the
|
||
// WRONG function), then regenerate a fresh unique sig. ----
|
||
let (images, unreadable_libs) = preload_images(prof, &cat, target);
|
||
// Named loudly: every catalogue entry pointing at one of these will resolve nowhere and ship as
|
||
// `SigDrifted`, whose detail says a signature drifted in the target — a false CAUSE for a file that
|
||
// was never opened. The reason has to reach the log even though it cannot reach every flag detail.
|
||
if !unreadable_libs.is_empty() {
|
||
eprintln!(
|
||
" WARNING {} target librar{} PRESENT but unreadable — every catalogue entry naming one \
|
||
scans nothing and is flagged as drifted, which is not why it failed: {}",
|
||
unreadable_libs.len(),
|
||
if unreadable_libs.len() == 1 {
|
||
"y is"
|
||
} else {
|
||
"ies are"
|
||
},
|
||
unreadable_libs.join(", ")
|
||
);
|
||
}
|
||
// The target must be a build DIRECTORY that `find_file` can search (a bare `.so` path is NOT searched —
|
||
// read_dir on a file yields nothing). Without this, a mistyped/file target loads zero images and the
|
||
// derive silently emits an all-flagged, near-empty release that even clears the live gate. Fail loudly
|
||
// instead — unless the catalogue is offset-only (no sigs to locate), where empty images is legitimate.
|
||
ensure!(
|
||
images.contains_key(prof.server_lib) || cat.iter().all(|f| linux_sigs(f).is_empty()),
|
||
"target {} loaded no {} — {}",
|
||
label_of(target),
|
||
prof.server_lib,
|
||
// Two very different causes, and telling the operator the wrong one costs an hour: a mistyped or
|
||
// file-shaped target searches nothing, while a present-but-corrupt server lib is a broken input.
|
||
if unreadable_libs
|
||
.iter()
|
||
.any(|u| u.starts_with(prof.server_lib))
|
||
{
|
||
"it is PRESENT but failed to parse (see the warning above) — truncated, or not an ELF"
|
||
} else {
|
||
"pass the build DIRECTORY (a bare .so path is not searched)"
|
||
}
|
||
);
|
||
|
||
// Sample the newest reference builds (there a sig still lands on the real function) to reject a target
|
||
// hit that is structurally a different function; see build_ref_fingerprints.
|
||
let ref_dirs: Vec<PathBuf> = ref_builds
|
||
.iter()
|
||
.rev()
|
||
.take(REF_BUILDS)
|
||
.map(|(_, p)| p.clone())
|
||
.collect();
|
||
let (ref_fps, ref_majority) = build_ref_fingerprints(prof, cmodel, &cat, &ref_dirs);
|
||
let dbg = std::env::var("SOURCE2ROSETTA_DBG").is_ok();
|
||
|
||
// Each catalogue function resolves independently — locate its candidates, verify the hit against the
|
||
// reference fingerprints, and emit a fresh sig — all read-only over images/ref_fps, each doing many
|
||
// whole-binary memchr scans. Resolve across threads (capturing any SOURCE2ROSETTA_DBG line), then apply in
|
||
// catalogue order so gd (name-keyed), the counts and the flag list are byte-identical to a serial run.
|
||
// Built once over the whole image set and shared read-only across the resolution threads: both halves
|
||
// are whole-image passes, so recomputing per function would dominate the derive.
|
||
let ident = Identity::of(&images);
|
||
let verify = Verify {
|
||
ref_fps,
|
||
ref_majority,
|
||
ident: &ident,
|
||
};
|
||
let sig_items = resolve_signatures(prof, &cat, &images, &verify, cmodel, dbg);
|
||
let (mut sig_ok, mut sig_flag, unverified, abi_drift, mut abi_shapes) =
|
||
apply_sig_items(sig_items, &mut gd);
|
||
|
||
// ---- one full-corpus vtable pass feeds BOTH offset chaining and vtable-anchoring recovery.
|
||
// Both need each class's vtable fingerprinted across every build; doing it once (over the
|
||
// union of the classes they need) halves the work versus two separate passes. ----
|
||
// still-flagged functions whose class has a target vtable = virtual-anchoring candidates
|
||
let virt_funcs: Vec<(&Func, String)> = cat
|
||
.iter()
|
||
.filter(|f| !linux_sigs(f).is_empty() && sig_flag.iter().any(|n| n == &f.name))
|
||
.map(|f| (f, class_of(&f.name).to_string()))
|
||
.filter(|(_, c)| {
|
||
images
|
||
.values()
|
||
.any(|im| rtti::find_vtable(im, c, prof.max_vtable_slots).is_some())
|
||
})
|
||
.collect();
|
||
|
||
let classes: Vec<String> = vfuncs
|
||
.iter()
|
||
.map(|f| f.class.clone())
|
||
.chain(virt_funcs.iter().map(|(_, c)| c.clone()))
|
||
.collect::<BTreeSet<_>>()
|
||
.into_iter()
|
||
.collect();
|
||
|
||
// per-build vtable data (fingerprints + addresses + resolved anchors): from the MODEL (only the
|
||
// target binary is read; past builds are synthesised from the distilled signals) or by
|
||
// fingerprinting every reference build's binaries.
|
||
let (per_build, target_idx) =
|
||
build_per_build_vtables(prof, cmodel, ref_builds, target, &classes, &virt_funcs);
|
||
// vtable-slot alignment across consecutive builds: the model ships past hops; the final hop
|
||
// (latest model build -> target) is computed from the target's real vtable fingerprints.
|
||
let hops = build_hops(cmodel, &per_build, target_idx, &classes);
|
||
|
||
// vtable-anchoring recovery for changed virtual functions (shares the pass above)
|
||
let vrec = recover_virtual_sigs(
|
||
prof,
|
||
&virt_funcs,
|
||
&per_build,
|
||
&hops,
|
||
target_idx,
|
||
&images,
|
||
&mut gd,
|
||
);
|
||
if !vrec.is_empty() {
|
||
sig_ok += vrec.len();
|
||
// Re-measured AT THE ADDRESS RECOVERY LANDED ON. The resolver ships no footprint for a flagged
|
||
// name precisely because recovery moves it, so this is what puts one back — measured where the
|
||
// signature now points rather than where it used to.
|
||
for (name, lib, addr) in &vrec {
|
||
if let Some(sh) = images.get(*lib).and_then(|im| abi::abi_shape(im, *addr)) {
|
||
abi_shapes.insert(name.clone(), shipped_abi(sh));
|
||
}
|
||
}
|
||
let names: Vec<&str> = vrec.iter().map(|(n, ..)| n.as_str()).collect();
|
||
sig_flag.retain(|n| !names.contains(&n.as_str()));
|
||
eprintln!(
|
||
" {} drifted signatures RECOVERED via vtable anchoring: {}",
|
||
vrec.len(),
|
||
names.join(", ")
|
||
);
|
||
}
|
||
|
||
// ---- string-anchor recovery: functions carrying a distinctive search-string (a sig-list anchor)
|
||
// are relocated in the target by that string and a fresh sig emitted — re-resolved per build,
|
||
// so the anchor survives byte-sig drift. Recovers anchors with no byte sig AND still-flagged
|
||
// ones that also carry an anchor. ----
|
||
let arec = recover_by_string_anchor(prof, &cat, &images, &mut gd);
|
||
if !arec.is_empty() {
|
||
sig_ok += arec.len();
|
||
// Same re-measurement, same reason. String anchors are server-lib by construction.
|
||
if let Some(im) = images.get(prof.server_lib) {
|
||
for (name, addr) in &arec {
|
||
if let Some(sh) = abi::abi_shape(im, *addr) {
|
||
abi_shapes.insert(name.clone(), shipped_abi(sh));
|
||
}
|
||
}
|
||
}
|
||
let names: Vec<&str> = arec.iter().map(|(n, _)| n.as_str()).collect();
|
||
sig_flag.retain(|n| !names.contains(&n.as_str()));
|
||
eprintln!(
|
||
" {} functions RECOVERED by string anchor: {}",
|
||
arec.len(),
|
||
names.join(", ")
|
||
);
|
||
}
|
||
|
||
// ---- offsets: chain from dated corpus reference builds to the target, then vote ----
|
||
let (off_ok, off_flag) = derive_offsets(&vfuncs, &hops, &per_build, target_idx, &mut gd);
|
||
|
||
let core = emitter.render(&gd);
|
||
eprintln!(
|
||
"gamedata for {}: {sig_ok} signatures + {off_ok} offsets emitted",
|
||
label_of(target)
|
||
);
|
||
if !sig_flag.is_empty() {
|
||
eprintln!(
|
||
" {} signatures FLAGGED (drifted): {}",
|
||
sig_flag.len(),
|
||
sig_flag.join(", ")
|
||
);
|
||
}
|
||
if !off_flag.is_empty() {
|
||
let rendered: Vec<String> = off_flag
|
||
.iter()
|
||
.map(|f| format!("{} ({})", f.name, f.detail))
|
||
.collect();
|
||
eprintln!(
|
||
" {} offsets FLAGGED (review): {}",
|
||
off_flag.len(),
|
||
rendered.join(", ")
|
||
);
|
||
}
|
||
// Scoped to the drift reason: this vec also carries the name-contradiction refusals, which are a
|
||
// different fact and are reported where they are found.
|
||
let drifted: Vec<&model::Flagged> = abi_drift
|
||
.iter()
|
||
.filter(|f| f.reason == model::FlagReason::AbiDrift)
|
||
.collect();
|
||
if !drifted.is_empty() {
|
||
// A prototype-drift WARNING (sig still emitted): the target's ABI shape differs from the model's
|
||
// consensus — the loader-hook seam the byte-sig can't see, caught forward without the corpus.
|
||
eprintln!(
|
||
" {} signatures with ABI DRIFT vs model consensus (kept; review prototype): {}",
|
||
drifted.len(),
|
||
drifted
|
||
.iter()
|
||
.map(|f| format!("{} ({})", f.name, f.detail))
|
||
.collect::<Vec<_>>()
|
||
.join(", ")
|
||
);
|
||
}
|
||
// The flag list: sig-drifted + offset-review + abi-drift, for the monolith's unresolved tier.
|
||
let mut flagged: Vec<model::Flagged> = sig_flag
|
||
.into_iter()
|
||
.map(|n| {
|
||
model::Flagged::new(
|
||
n,
|
||
model::FlagReason::SigDrifted,
|
||
"no unique/recovered signature in target",
|
||
)
|
||
})
|
||
.collect();
|
||
flagged.extend(off_flag);
|
||
flagged.extend(abi_drift);
|
||
// ---- the completeness sweep, and it is the thing that makes `unresolved` a CONTRACT rather than a
|
||
// by-product. Every pass above flags what IT could not place: the sig resolver flags a drifted
|
||
// signature, the offset pass flags a low-confidence vote. But a name reaches those passes only if
|
||
// it has the input they consume, and one class of entry has neither — an ANCHOR-ONLY catalogue
|
||
// entry (476 of CS2's 1,601) never enters the sig resolver at all, so when its anchor finds zero
|
||
// or several xref hits it lands in no tier and no flag list: it simply stops existing, which is
|
||
// the one outcome README.md ("it never just disappears") and CONTRIBUTING.md both rule out.
|
||
// Sweeping the catalogue itself, rather than adding a fourth per-pass flag, is what makes the
|
||
// claim hold for the NEXT locator kind too. ----
|
||
let accounted: HashSet<&str> = gd
|
||
.entries
|
||
.keys()
|
||
.map(String::as_str)
|
||
.chain(flagged.iter().map(|f| f.name.as_str()))
|
||
.collect();
|
||
let by_name: HashMap<&str, &Func> = cat.iter().map(|f| (f.name.as_str(), f)).collect();
|
||
let mut vanished: Vec<&str> = by_name
|
||
.keys()
|
||
.copied()
|
||
.filter(|n| !accounted.contains(n))
|
||
.collect();
|
||
vanished.sort_unstable();
|
||
if !vanished.is_empty() {
|
||
eprintln!(
|
||
" {} catalogue entries produced no locator and no flag — recorded as unresolved: {}",
|
||
vanished.len(),
|
||
vanished.join(", ")
|
||
);
|
||
flagged.extend(vanished.into_iter().map(|n| {
|
||
model::Flagged::new(
|
||
n,
|
||
model::FlagReason::Unresolved,
|
||
unresolvable_because(by_name[n]),
|
||
)
|
||
}));
|
||
}
|
||
// Every catalogue entry's anchors, whether or not a byte sig located it. `recover_by_string_anchor`
|
||
// uses them only as a FALLBACK locator; this ships them as a supplement, which is a different job — an
|
||
// entry that resolved perfectly still benefits from a second locator with a different failure mode.
|
||
let anchors: BTreeMap<String, Vec<String>> = cat
|
||
.iter()
|
||
.filter_map(|f| {
|
||
let a: Vec<String> = string_anchors(f).into_iter().map(str::to_string).collect();
|
||
(!a.is_empty()).then(|| (f.name.clone(), a))
|
||
})
|
||
.collect();
|
||
Ok(Derived {
|
||
core,
|
||
flagged,
|
||
unverified: unverified.into_iter().collect(),
|
||
abi: abi_shapes,
|
||
anchors,
|
||
})
|
||
}
|
||
|
||
/// One build's vtable data, shared by offset chaining and vtable-anchoring recovery.
|
||
#[derive(Default)]
|
||
struct BuildVtables {
|
||
date: String,
|
||
fps: VtableFps, // class -> slot fingerprints (for chaining)
|
||
addrs: HashMap<String, Vec<u64>>, // class -> slot addresses (to read the located slot)
|
||
resolved: HashMap<String, u64>, // anchoring candidate name -> resolved address
|
||
/// Libraries present in the build but unparseable. Empty on a healthy build; non-empty means this
|
||
/// build's fingerprints are partial, which silently breaks anchor chains through it.
|
||
unreadable: Vec<String>,
|
||
/// True when no library loaded at all — the build contributes nothing but still occupies an index.
|
||
empty: bool,
|
||
}
|
||
|
||
/// Recover functions by a distinctive string they reference (a `string-anchor` locator from a curated sig
|
||
/// list). Unlike byte sigs, the anchor string is stable across builds, so this re-resolves
|
||
/// per build: for each catalogue function with a string-anchor not already located by a byte sig, take
|
||
/// the *unique* function referencing that string in the target and emit a fresh sig there. A string
|
||
/// referenced by ≥2 functions is not distinctive enough → skipped (stays flagged). Server-only (all
|
||
/// curated anchors are server-lib). Returns each recovered name WITH the address it was relocated to —
|
||
/// the caller re-measures the footprint there, because the one measured before recovery belonged to an
|
||
/// address this build rejected.
|
||
fn recover_by_string_anchor(
|
||
prof: &GameProfile,
|
||
cat: &[Func],
|
||
target_images: &HashMap<String, CodeImage>,
|
||
gd: &mut model::Gamedata,
|
||
) -> Vec<(String, u64)> {
|
||
let anchored: Vec<&Func> = cat
|
||
.iter()
|
||
.filter(|f| !string_anchors(f).is_empty())
|
||
.collect();
|
||
let Some(img) = target_images.get(prof.server_lib) else {
|
||
return Vec::new();
|
||
};
|
||
if anchored.is_empty() {
|
||
return Vec::new();
|
||
}
|
||
let idx = xref::XrefIndex::build(img);
|
||
let mut recovered = Vec::new();
|
||
let mut unreachable = 0usize;
|
||
for f in anchored {
|
||
if gd.entries.contains_key(&f.name) {
|
||
continue; // already located by a byte sig — the anchor is just a backup
|
||
}
|
||
for s in string_anchors(f) {
|
||
let hits = xref::funcs_using_string(img, &idx, s);
|
||
let [addr] = hits.as_slice() else { continue };
|
||
// "Exactly one function references it" is attribution by `containing_func` — nearest entry
|
||
// at-or-below — and these binaries do not support that on its own: CS2 strips `.eh_frame`
|
||
// from game code, so a `[entry, next_entry)` range routinely spans a real function plus
|
||
// unindexed neighbours and inherits their strings. `attach_derived_anchors` spends two
|
||
// conditions on exactly this hazard when it DERIVES an anchor; using one to locate a
|
||
// function is the same claim in the other direction and gets the same test: every
|
||
// instruction that loads the string must sit inside the candidate's own flow-reachable
|
||
// code. Where it does not, the real referrer is unindexed and this "unique" hit is its
|
||
// preceding neighbour — a signature that resolves, validates live, and names the wrong
|
||
// function, shipped into the guaranteed tier. Dropping it is the honest degradation.
|
||
let Some((reach, _)) = reachable_strings(img, *addr) else {
|
||
continue;
|
||
};
|
||
let attributable = img
|
||
.find_bytes(s.as_bytes())
|
||
.into_iter()
|
||
.flat_map(|va| idx.refs_to(va))
|
||
.all(|ip| reach.contains(ip));
|
||
if !attributable {
|
||
unreachable += 1;
|
||
continue;
|
||
}
|
||
if let Some(sig) = emit::make_sig(img, *addr, CORE_SIG_CAP) {
|
||
gd.set_signature(f.name.clone(), "server", sig);
|
||
recovered.push((f.name.clone(), *addr));
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
if unreachable > 0 {
|
||
eprintln!(
|
||
" string-anchor recovery: {unreachable} anchors REJECTED — the string is loaded from \
|
||
outside the candidate's reachable code, so the unique-referrer hit is a neighbouring \
|
||
function rather than the one the anchor names"
|
||
);
|
||
}
|
||
recovered
|
||
}
|
||
|
||
/// Chain each anchor's slot forward through the vtable-alignment hops `hv` to the target build, then take
|
||
/// a RECENCY-WEIGHTED vote on the resulting target slot: each anchor's vote is weighted by its build index,
|
||
/// so a recent anchor — which chains through fewer hops and reflects the current build's slot convention —
|
||
/// outweighs a distant one whose value may predate a layout shift or come from a differently-indexed tool.
|
||
/// An offset consistent across history is unaffected, since every anchor chains to the same slot.
|
||
/// Returns the winning (slot, confidence%), or None if no anchor chained through.
|
||
fn chain_and_vote(
|
||
anchors: &[(usize, usize)],
|
||
hv: &[Vec<Option<usize>>],
|
||
target_idx: usize,
|
||
) -> Option<(usize, u64)> {
|
||
let mut votes: HashMap<usize, u64> = HashMap::new();
|
||
for &(a, off0) in anchors {
|
||
let mut pos = Some(off0);
|
||
for t in a..target_idx {
|
||
pos = match hv.get(t) {
|
||
Some(m) if !m.is_empty() => pos.and_then(|p| m.get(p).copied().flatten()),
|
||
_ => None,
|
||
};
|
||
if pos.is_none() {
|
||
break;
|
||
}
|
||
}
|
||
if let Some(p) = pos {
|
||
*votes.entry(p).or_default() += a as u64 + 1;
|
||
}
|
||
}
|
||
let total: u64 = votes.values().sum();
|
||
let (&slot, &agree) = votes
|
||
.iter()
|
||
.max_by_key(|e| (*e.1, std::cmp::Reverse(*e.0)))?;
|
||
Some((slot, 100 * agree / total.max(1)))
|
||
}
|
||
|
||
/// Time-spread vtable anchors: walk reference builds `[0, target_idx)`, keeping at most one anchor per
|
||
/// 15-build window, where `slot_of(build)` yields the function's slot in that build. The two vtable paths
|
||
/// (offset chaining + drifted-sig recovery) differ only in HOW they read a build's slot, so this captures the
|
||
/// shared spacing loop; the closure supplies the per-build read.
|
||
fn spread_anchors(
|
||
per_build: &[BuildVtables],
|
||
target_idx: usize,
|
||
mut slot_of: impl FnMut(&BuildVtables) -> Option<usize>,
|
||
) -> Vec<(usize, usize)> {
|
||
let mut anchors: Vec<(usize, usize)> = Vec::new();
|
||
for (i, bv) in per_build.iter().enumerate().take(target_idx) {
|
||
if anchors.last().is_some_and(|&(li, _)| i - li < 15) {
|
||
continue;
|
||
}
|
||
if let Some(off) = slot_of(bv) {
|
||
anchors.push((i, off));
|
||
}
|
||
}
|
||
anchors
|
||
}
|
||
|
||
/// Derive each catalogued vtable offset for the target build: chain its dated reference offsets forward
|
||
/// through the per-build slot-alignment hops, then take a recency-weighted vote. Only a high-confidence
|
||
/// chain is emitted; a weak one is flagged, and one with no usable reference at all is recorded as
|
||
/// `unresolved` rather than dropped.
|
||
fn derive_offsets(
|
||
vfuncs: &[VtFunc],
|
||
hops: &HashMap<&str, Vec<Vec<Option<usize>>>>,
|
||
per_build: &[BuildVtables],
|
||
target_idx: usize,
|
||
gd: &mut model::Gamedata,
|
||
) -> (u32, Vec<model::Flagged>) {
|
||
let mut off_ok = 0;
|
||
let mut off_flag: Vec<model::Flagged> = Vec::new();
|
||
for f in vfuncs {
|
||
let hv = &hops[f.class.as_str()];
|
||
let anchors = spread_anchors(per_build, target_idx, |bv| {
|
||
let o0 = offset_at(&f.timeline, &bv.date)? as usize;
|
||
(o0 < bv.fps.get(&f.class)?.len()).then_some(o0)
|
||
});
|
||
if anchors.is_empty() {
|
||
// Nothing to chain from: the offset exceeds the vtable (a struct FIELD offset like
|
||
// m_iHitGroup), the class has no locatable vtable, or — the common case for a fresh
|
||
// contribution — the value is dated at or after the newest reference build. We deliberately
|
||
// do NOT carry the last known value forward: CS2 inherits Source-1 class NAMES (CAK47,
|
||
// CBaseButton) whose LAYOUTS differ, so an ecosystem date-src offset for a coincident name is
|
||
// a different game's value and would ship a wrong offset. Drop it, but VISIBLY — record it so
|
||
// `unresolved` is the complete catalogue picture the consumer contract claims, and a
|
||
// contributor gets a reason instead of nothing.
|
||
off_flag.push(model::Flagged::new(
|
||
f.name.clone(),
|
||
model::FlagReason::Unresolved,
|
||
"no chainable anchor (offset exceeds the vtable, class has no vtable, or the value \
|
||
post-dates every reference build)",
|
||
));
|
||
continue;
|
||
}
|
||
// Recency-weighted vote (see chain_and_vote): only a high-confidence chain reaches the shipped
|
||
// gamedata. A low-confidence chain over a cross-game catalogue is usually a coincident class NAME
|
||
// (CS2 shares CAK47/CBaseEntity with GoldSrc) whose offset chained by luck — flag, never emit.
|
||
match chain_and_vote(&anchors, hv, target_idx) {
|
||
Some((pred, conf)) if conf >= 80 => {
|
||
gd.set_offset(f.name.clone(), pred as i64);
|
||
// The class whose vtable this slot indexes — half the locator, since an index alone locates
|
||
// nothing. It is `class_of(f.name)`: `vtable_offset_timelines` builds `VtFunc::class` that
|
||
// way and both `hops` and `bv.fps` are keyed by it, so the class the chain walked and the
|
||
// class in the name are one fact, not two. Emitting it saves the consumer a name split; it
|
||
// does not add information the name lacks.
|
||
gd.set_class(f.name.clone(), f.class.clone());
|
||
off_ok += 1;
|
||
}
|
||
Some((pred, conf)) => {
|
||
off_flag.push(model::Flagged::new(
|
||
f.name.clone(),
|
||
model::FlagReason::OffsetLowConf,
|
||
format!("offset {pred}, conf {conf}% — not emitted"),
|
||
));
|
||
}
|
||
None => off_flag.push(model::Flagged::new(
|
||
f.name.clone(),
|
||
model::FlagReason::Unresolved,
|
||
"no vote",
|
||
)),
|
||
}
|
||
}
|
||
(off_ok, off_flag)
|
||
}
|
||
|
||
/// For each class, its vtable slot-fingerprints + slot addresses from the FIRST library (in the given
|
||
/// precedence order) whose primary vtable holds it — the deterministic, correctness-critical "first lib
|
||
/// holding the class wins" rule. Extracted so the model and binary paths of
|
||
/// `build_per_build_vtables` implement that precedence exactly once.
|
||
fn target_vtables<'a>(
|
||
prof: &GameProfile,
|
||
imgs: impl IntoIterator<Item = &'a CodeImage>,
|
||
classes: &[String],
|
||
) -> (
|
||
HashMap<String, Vec<Option<Vec<u32>>>>,
|
||
HashMap<String, Vec<u64>>,
|
||
) {
|
||
let imgs: Vec<&CodeImage> = imgs.into_iter().collect();
|
||
let (mut fps, mut addrs) = (HashMap::new(), HashMap::new());
|
||
for c in classes {
|
||
for img in &imgs {
|
||
if let Some(vt) = rtti::find_vtable(img, c, prof.max_vtable_slots) {
|
||
fps.insert(c.clone(), slot_fingerprints(img, &vt.slots));
|
||
addrs.insert(c.clone(), vt.slots);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
(fps, addrs)
|
||
}
|
||
|
||
/// Per-build vtable data (fingerprints + slot addresses + resolved anchors) plus the target's index —
|
||
/// the input to both offset chaining and vtable-anchoring recovery. From the MODEL (only the target
|
||
/// binary is read; past builds are synthesised from the distilled slot counts + anchor slots so the
|
||
/// downstream is byte-identical) or by fingerprinting every reference build's binaries.
|
||
fn build_per_build_vtables(
|
||
prof: &GameProfile,
|
||
cmodel: Option<&CorpusModel>,
|
||
ref_builds: Vec<(String, PathBuf)>,
|
||
target: &Path,
|
||
classes: &[String],
|
||
virt_funcs: &[(&Func, String)],
|
||
) -> (Vec<BuildVtables>, usize) {
|
||
if let Some(m) = cmodel {
|
||
// The one build we DO read — the target's real vtable slots + fingerprints. Load the WHOLE
|
||
// profile lib set rather than reusing `images`: that map is preloaded from the libraries the
|
||
// SIGNATURE half names, and the catalogue's `library` hint is community-sourced and routinely
|
||
// wrong about which module a class lives in (`CSoundSystem` and `CVPhys2World` are both
|
||
// catalogued "server" while their vtables are in `libsoundsystem` / `libvphysics2`). Searching
|
||
// only the sig-named subset silently dropped those offsets from every release built on this
|
||
// path while the `--corpus` path shipped them; the two paths must not disagree. The binary
|
||
// branch below already loads the full set for exactly this reason.
|
||
//
|
||
// `load_build_images` yields the libs in `prof.libs` precedence order, which is what keeps the
|
||
// choice deterministic: a class whose vtable appears in more than one lib must resolve to the
|
||
// same image here as everywhere else, and the first lib holding it wins.
|
||
let timgs = load_build_images(prof, target);
|
||
let (tf, ta) = target_vtables(prof, &timgs, classes);
|
||
// The target's own (real, newest) date — offset_at(timeline, target_date) then returns the latest
|
||
// timeline entry (the target is newer than every corpus build by the forward-only guard); the
|
||
// target's position is tracked explicitly by `target_idx` below.
|
||
let target_bv = BuildVtables {
|
||
date: build_date(&label_of(target)),
|
||
fps: tf,
|
||
addrs: ta,
|
||
resolved: HashMap::new(),
|
||
// synthetic: reconstructed from the model, not loaded from images, so there is nothing to report
|
||
..Default::default()
|
||
};
|
||
// past builds: encode the model's distilled slot counts + anchor slots as a synthetic
|
||
// BuildVtables so the downstream is byte-identical — fps length = slot_count (bounds checks),
|
||
// addrs = slot indices, resolved = the anchor slot (so `position(resolved in addrs)` recovers it).
|
||
let mut pb: Vec<BuildVtables> = m
|
||
.builds
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(i, bm)| {
|
||
let (mut fps, mut addrs) = (HashMap::new(), HashMap::new());
|
||
for c in classes {
|
||
let n = m
|
||
.slot_counts
|
||
.get(c)
|
||
.and_then(|v| v.get(i))
|
||
.copied()
|
||
.unwrap_or(0);
|
||
if n > 0 {
|
||
fps.insert(c.clone(), vec![None; n]);
|
||
addrs.insert(c.clone(), (0..n as u64).collect());
|
||
}
|
||
}
|
||
let mut resolved = HashMap::new();
|
||
for (f, _) in virt_funcs {
|
||
if let Some(slot) = m
|
||
.resolved_slot
|
||
.get(&f.name)
|
||
.and_then(|v| v.get(i))
|
||
.and_then(|s| s.get())
|
||
{
|
||
resolved.insert(f.name.clone(), slot as u64);
|
||
}
|
||
}
|
||
BuildVtables {
|
||
date: bm.date.clone(),
|
||
fps,
|
||
addrs,
|
||
resolved,
|
||
..Default::default() // synthetic (model-derived): no images were loaded
|
||
}
|
||
})
|
||
.collect();
|
||
pb.push(target_bv);
|
||
let ti = pb.len() - 1;
|
||
(pb, ti)
|
||
} else {
|
||
let mut builds = ref_builds;
|
||
// the target's real (newest) date — see the model branch above; `ti` is its explicit position marker.
|
||
builds.push((build_date(&label_of(target)), target.to_path_buf()));
|
||
let ti = builds.len() - 1;
|
||
let pb = parallel_map(&builds, default_threads(None), |(date, dir)| {
|
||
let imgs = load_build_images(prof, dir);
|
||
let (fps, addrs) = target_vtables(prof, &imgs, classes);
|
||
let mut resolved = HashMap::new();
|
||
for (f, _) in virt_funcs {
|
||
if let Some(a) = resolve_unique(f, &imgs) {
|
||
resolved.insert(f.name.clone(), a);
|
||
}
|
||
}
|
||
BuildVtables {
|
||
date: date.clone(),
|
||
fps,
|
||
addrs,
|
||
resolved,
|
||
..Default::default() // built from already-loaded images; load reporting happens there
|
||
}
|
||
});
|
||
(pb, ti)
|
||
}
|
||
}
|
||
|
||
/// Vtable-slot alignment hops per class: the model ships the past consecutive-build hops and the final
|
||
/// hop (latest model build -> target) is computed from the target's real vtable fingerprints; the binary
|
||
/// path aligns every consecutive pair directly. Keys borrow `classes`.
|
||
fn build_hops<'a>(
|
||
cmodel: Option<&CorpusModel>,
|
||
per_build: &[BuildVtables],
|
||
target_idx: usize,
|
||
classes: &'a [String],
|
||
) -> HashMap<&'a str, Vec<Vec<Option<usize>>>> {
|
||
classes
|
||
.iter()
|
||
.map(|c| {
|
||
let hv = if let Some(m) = cmodel {
|
||
let mut h: Vec<Vec<Option<usize>>> = m
|
||
.hops
|
||
.get(c)
|
||
.map(|v| v.iter().map(|inner| inner.to_options()).collect())
|
||
.unwrap_or_default();
|
||
let fh = match (m.latest_vtable_fps.get(c), per_build[target_idx].fps.get(c)) {
|
||
(Some(a), Some(b)) => nw_align(a, b),
|
||
_ => Vec::new(),
|
||
};
|
||
h.push(fh);
|
||
h
|
||
} else {
|
||
per_build
|
||
.windows(2)
|
||
.map(|w| match (w[0].fps.get(c), w[1].fps.get(c)) {
|
||
(Some(a), Some(b)) => nw_align(a, b),
|
||
_ => Vec::new(),
|
||
})
|
||
.collect()
|
||
};
|
||
(c.as_str(), hv)
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
/// Recover drifted VIRTUAL functions by vtable slot — content-INDEPENDENT, so it works even when the
|
||
/// function changed so much no fingerprint matches. For each candidate: find builds where its signature
|
||
/// still resolved *to a slot of its class vtable* (that slot index is its offset, and requiring a real slot
|
||
/// auto-rejects decoy matches), chain the offset forward to the target through the order-preserving vtable
|
||
/// alignments, vote on the target slot, then read `vtable[offset]` in the target and emit a sig there.
|
||
/// Consumes the shared vtable pass. Returns each recovered name with the LIBRARY and address it was
|
||
/// relocated to, so the caller can re-measure the footprint there — see `recover_by_string_anchor`.
|
||
fn recover_virtual_sigs(
|
||
prof: &GameProfile,
|
||
virt_funcs: &[(&Func, String)],
|
||
per_build: &[BuildVtables],
|
||
hops: &HashMap<&str, Vec<Vec<Option<usize>>>>,
|
||
target_idx: usize,
|
||
target_images: &HashMap<String, CodeImage>,
|
||
gd: &mut model::Gamedata,
|
||
) -> Vec<(String, &'static str, u64)> {
|
||
let mut recovered = Vec::new();
|
||
for (f, class) in virt_funcs {
|
||
let Some(hv) = hops.get(class.as_str()) else {
|
||
continue;
|
||
};
|
||
// anchors: time-spread builds where the sig resolved to a slot of this class's vtable
|
||
let anchors = spread_anchors(per_build, target_idx, |bv| {
|
||
let &addr = bv.resolved.get(&f.name)?;
|
||
bv.addrs
|
||
.get(class.as_str())?
|
||
.iter()
|
||
.position(|&a| a == addr)
|
||
});
|
||
if anchors.is_empty() {
|
||
continue; // never resolved to a slot -> can't anchor
|
||
}
|
||
// chain each anchor to the target and recency-weight-vote the target slot
|
||
let Some((off, conf)) = chain_and_vote(&anchors, hv, target_idx) else {
|
||
continue;
|
||
};
|
||
if conf < 80 {
|
||
continue; // low agreement -> leave flagged
|
||
}
|
||
// read the located slot in the target and emit a sig there. All CS2 .so link at vaddr 0, so
|
||
// is_code(addr) is ambiguous across libraries — resolve the class's OWN vtable image
|
||
// (find_vtable is class-specific) so make_sig reads the right bytes, deterministically, and
|
||
// the emitted library is correct rather than a hardcoded "server".
|
||
if let Some(&addr) = per_build[target_idx]
|
||
.addrs
|
||
.get(class.as_str())
|
||
.and_then(|s| s.get(off))
|
||
{
|
||
let lib = prof.libs.iter().copied().find(|&l| {
|
||
target_images
|
||
.get(l)
|
||
.and_then(|im| rtti::find_vtable(im, class, prof.max_vtable_slots))
|
||
.is_some_and(|vt| vt.slots.get(off) == Some(&addr))
|
||
});
|
||
if let Some(lib) = lib
|
||
&& let Some(sig) = emit::make_sig(&target_images[lib], addr, CORE_SIG_CAP)
|
||
{
|
||
gd.set_signature(f.name.clone(), lib_name_from_file(lib), sig);
|
||
recovered.push((f.name.clone(), lib, addr));
|
||
}
|
||
}
|
||
}
|
||
recovered
|
||
}
|
||
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
// §8 · BACKFILL (cross-build name/offset timelines)
|
||
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||
|
||
#[derive(Deserialize)]
|
||
struct BackfillItem {
|
||
name: String,
|
||
#[serde(default)]
|
||
tier: String,
|
||
#[serde(default)]
|
||
anchor: Option<String>, // a distinctive string it references (drives the binary string-anchor half)
|
||
#[serde(default)]
|
||
class: Option<String>, // drives the model-hops half (with slot)
|
||
#[serde(default)]
|
||
slot: Option<i64>,
|
||
}
|
||
|
||
/// Chain an offset's vtable slot BACKWARD through a class's per-build-pair alignment hops, from its known
|
||
/// latest-build slot to every earlier build — the model-only history for a vtable method (no binaries).
|
||
/// `hops[i]` aligns build i -> i+1 (indexed by OLD slot); inverting each hop walks the slot back in time.
|
||
fn hops_timeline(
|
||
hops: &[Vec<Option<usize>>],
|
||
latest_slot: usize,
|
||
latest_count: usize,
|
||
) -> Vec<Option<usize>> {
|
||
let n = hops.len() + 1;
|
||
let mut tl = vec![None; n];
|
||
// Guard the tip like `vtable_slot_timeline`: a slot beyond the class's latest-build vtable is NOT located
|
||
// (seed None), so an out-of-range slot doesn't count as "located at the newest build" and spuriously
|
||
// inflate this method's coverage in the cov() tiebreak against the bounds-checked binary method.
|
||
tl[n - 1] = (latest_slot < latest_count).then_some(latest_slot);
|
||
for i in (0..hops.len()).rev() {
|
||
if let Some(target) = tl[i + 1] {
|
||
tl[i] = hops[i].iter().position(|&m| m == Some(target));
|
||
}
|
||
}
|
||
tl
|
||
}
|
||
|
||
/// The class's per-build vtable-slot fingerprints (None where the class is absent in that build). Chained
|
||
/// to walk one latest-build slot back through time.
|
||
type VtSeq = Vec<Option<Vec<Option<Vec<u32>>>>>;
|
||
|
||
/// Track a class's vtable slot BACKWARD across the raw builds by chaining the order-preserving vtable
|
||
/// alignments (the `chain` logic, for one latest-build slot). Reliable where model `hops` aren't
|
||
/// available — adjacent-build alignment is near-perfect — and it covers ANY class, not just model-tracked
|
||
/// ones. This is the binary path for offset back-fill (the model-hops path is its binaries-free CI twin).
|
||
fn vtable_slot_timeline(seq: &VtSeq, tip_slot: usize) -> Vec<Option<usize>> {
|
||
let n = seq.len();
|
||
let mut tl = vec![None; n];
|
||
if n == 0 {
|
||
return tl;
|
||
}
|
||
tl[n - 1] = seq[n - 1]
|
||
.as_ref()
|
||
.filter(|f| tip_slot < f.len())
|
||
.map(|_| tip_slot);
|
||
for i in (0..n - 1).rev() {
|
||
if let (Some(target), Some(from), Some(to)) =
|
||
(tl[i + 1], seq[i].as_ref(), seq[i + 1].as_ref())
|
||
{
|
||
let map = nw_align(from, to); // from(build i) -> to(build i+1)
|
||
tl[i] = map.iter().position(|&m| m == Some(target));
|
||
}
|
||
}
|
||
tl
|
||
}
|
||
|
||
/// One raw build's back-fill signals: the unique anchor address per name + each needed class's vtable
|
||
/// slot fingerprints (for the vtable-chain method).
|
||
struct BuildBackfill {
|
||
anchor: Vec<Option<u64>>,
|
||
vt: HashMap<String, Vec<Option<Vec<u32>>>>,
|
||
}
|
||
|
||
/// Which timeline won for a back-filled name — a closed domain that also encodes how its values read: `Anchor`
|
||
/// timelines are ADDRESSES (formatted `{:#x}`), `Vtable`/`Hops` are vtable SLOTS (plain).
|
||
#[derive(Clone, Copy)]
|
||
enum BackfillMethod {
|
||
Anchor,
|
||
Vtable,
|
||
Hops,
|
||
}
|
||
|
||
impl BackfillMethod {
|
||
/// The report label.
|
||
fn as_str(self) -> &'static str {
|
||
match self {
|
||
BackfillMethod::Anchor => "anchor",
|
||
BackfillMethod::Vtable => "vtable",
|
||
BackfillMethod::Hops => "hops",
|
||
}
|
||
}
|
||
/// Timeline values are addresses (`{:#x}`), not vtable slots.
|
||
fn is_address(self) -> bool {
|
||
matches!(self, BackfillMethod::Anchor)
|
||
}
|
||
}
|
||
|
||
/// Back-fill cross-build history for extrapolated (T3) names — the historical-enrichment + community-PR
|
||
/// ingestion tool. An OFFSET name rides the model's `hops` (no binaries — the vtable-method PR path); a
|
||
/// SIG / self-named name rides the string-anchor locator over the raw builds. Reports per-name history
|
||
/// depth + consistency-since-first-appearance: the measure of how many T3 names graduate to first-class
|
||
/// (deep + consistent history == high confidence, regardless of T3 origin). The compact per-name history
|
||
/// it emits is the persistent timeline that survives deleting the binaries.
|
||
pub fn backfill_cmd(
|
||
prof: &GameProfile,
|
||
corpus: Option<&Path>,
|
||
corpus_model: Option<&Path>,
|
||
lib: &str,
|
||
names: &Path,
|
||
threads: Option<usize>,
|
||
out: Option<&Path>,
|
||
) -> Result<()> {
|
||
// A backfill needs a build source to resolve anchors against; with neither it would run to completion
|
||
// reporting every name as "not located" and exit 0 — a vacuous success. Fail loudly, as `produce` does.
|
||
ensure!(
|
||
corpus.is_some() || corpus_model.is_some(),
|
||
"backfill needs --corpus <binaries> or --corpus-model <model.json> — with neither there is nothing \
|
||
to resolve anchors against"
|
||
);
|
||
let items: Vec<BackfillItem> = serde_json::from_str(&std::fs::read_to_string(names)?)
|
||
.context("parse backfill names json")?;
|
||
let model: Option<CorpusModel> = match corpus_model {
|
||
Some(p) => {
|
||
Some(serde_json::from_str(&std::fs::read_to_string(p)?).context("parse corpus model")?)
|
||
}
|
||
None => None,
|
||
};
|
||
let model_dates: Vec<String> = model
|
||
.as_ref()
|
||
.map(|m| m.builds.iter().map(|b| b.date.clone()).collect())
|
||
.unwrap_or_default();
|
||
|
||
// The binary half: per raw build, locate each name by its anchor string AND fingerprint each needed
|
||
// class's vtable (for the vtable-chain method — the reliable, all-classes offset back-fill).
|
||
let builds = match corpus {
|
||
Some(c) => find_builds(prof, c)?,
|
||
None => Vec::new(),
|
||
};
|
||
let anchor_dates: Vec<String> = builds.iter().map(|d| build_date(&label_of(d))).collect();
|
||
let vt_needed: BTreeSet<String> = items
|
||
.iter()
|
||
.filter(|it| it.slot.is_some_and(|s| s >= 0))
|
||
.filter_map(|it| it.class.clone())
|
||
.collect();
|
||
let per_build: Vec<BuildBackfill> = if builds.is_empty() {
|
||
Vec::new()
|
||
} else {
|
||
parallel_map(&builds, default_threads(threads), |dir| {
|
||
let empty = BuildBackfill {
|
||
anchor: vec![None; items.len()],
|
||
vt: HashMap::new(),
|
||
};
|
||
let Some(img) = find_file(dir, lib, 8).and_then(|p| CodeImage::load(&p).ok()) else {
|
||
return empty;
|
||
};
|
||
let idx = xref::XrefIndex::build(&img);
|
||
let anchor = items
|
||
.iter()
|
||
.map(|it| {
|
||
it.anchor.as_deref().and_then(|a| {
|
||
match xref::funcs_using_string(&img, &idx, a).as_slice() {
|
||
[addr] => Some(*addr),
|
||
_ => None, // absent or ambiguous in this build
|
||
}
|
||
})
|
||
})
|
||
.collect();
|
||
// one RTTI sweep, keep the needed primary vtables' slot fingerprints
|
||
let mut vt = HashMap::new();
|
||
if !vt_needed.is_empty() {
|
||
for cv in rtti::enumerate_vtables(&img, prof.max_vtable_slots)
|
||
.into_iter()
|
||
.filter(|c| c.offset_to_top == 0 && vt_needed.contains(&c.name))
|
||
{
|
||
let fps = slot_fingerprints(&img, &cv.slots);
|
||
vt.insert(cv.name, fps);
|
||
}
|
||
}
|
||
BuildBackfill { anchor, vt }
|
||
})
|
||
};
|
||
|
||
let mut report = Vec::new();
|
||
let mut buckets = [0usize; 4]; // deep-first-class, recent-consistent, flaky, not-located
|
||
let cov = |tl: &[Option<u64>]| tl.iter().filter(|x| x.is_some()).count();
|
||
for (i, it) in items.iter().enumerate() {
|
||
// Compute every applicable timeline, then KEEP the one with the most cross-build coverage — so a
|
||
// self-named offset that anchors perfectly isn't regressed onto a weaker vtable chain, and a
|
||
// dict-exact offset with no self-string is rescued by the vtable chain.
|
||
let mut best: (BackfillMethod, &[String], Vec<Option<u64>>) = (
|
||
BackfillMethod::Anchor,
|
||
&anchor_dates,
|
||
(0..per_build.len())
|
||
.map(|b| per_build[b].anchor[i])
|
||
.collect(),
|
||
);
|
||
// vtable-chain over binaries (offset, any class the RTTI sweep captured)
|
||
if let (Some(cls), Some(slot)) = (it.class.as_deref(), it.slot.filter(|&s| s >= 0)) {
|
||
if per_build.iter().any(|b| b.vt.contains_key(cls)) {
|
||
let seq: VtSeq = per_build.iter().map(|b| b.vt.get(cls).cloned()).collect();
|
||
let tl: Vec<Option<u64>> = vtable_slot_timeline(&seq, slot as usize)
|
||
.into_iter()
|
||
.map(|o| o.map(|s| s as u64))
|
||
.collect();
|
||
if cov(&tl) > cov(&best.2) {
|
||
best = (BackfillMethod::Vtable, &anchor_dates, tl);
|
||
}
|
||
}
|
||
// model hops (binaries-free) — the CI twin; used if it beats the binary methods
|
||
if let Some(h) = model
|
||
.as_ref()
|
||
.and_then(|m| m.hops.get(cls))
|
||
.filter(|h| !h.is_empty())
|
||
{
|
||
let h: Vec<Vec<Option<usize>>> = h.iter().map(|inner| inner.to_options()).collect();
|
||
let latest_count = model
|
||
.as_ref()
|
||
.and_then(|m| m.slot_counts.get(cls))
|
||
.and_then(|v| v.last())
|
||
.copied()
|
||
.unwrap_or(0);
|
||
let tl: Vec<Option<u64>> = hops_timeline(&h, slot as usize, latest_count)
|
||
.into_iter()
|
||
.map(|o| o.map(|s| s as u64))
|
||
.collect();
|
||
if cov(&tl) > cov(&best.2) {
|
||
best = (BackfillMethod::Hops, &model_dates, tl);
|
||
}
|
||
}
|
||
}
|
||
let (method, dates, timeline) = best;
|
||
let n = dates.len();
|
||
let located: Vec<usize> = (0..n).filter(|&b| timeline[b].is_some()).collect();
|
||
let count = located.len();
|
||
let first = located.first().copied();
|
||
let span = first.map_or(0, |f| n - f);
|
||
let consistency = if span > 0 {
|
||
count as f64 / span as f64
|
||
} else {
|
||
0.0
|
||
};
|
||
let bucket = if count == 0 {
|
||
3
|
||
} else if count * 2 >= n && consistency >= 0.9 {
|
||
0 // deep, consistent history == first-class regardless of T3 origin
|
||
} else if consistency >= 0.9 {
|
||
1 // recent but consistent since it appeared
|
||
} else {
|
||
2 // flaky — the anchor drifts / isn't distinctive
|
||
};
|
||
buckets[bucket] += 1;
|
||
// compact history: only located builds -> value (vtable slot for hops, address for anchor).
|
||
let history: serde_json::Map<String, Value> = located
|
||
.iter()
|
||
.filter_map(|&b| {
|
||
timeline[b].map(|v| {
|
||
// hops/vtable timelines are vtable SLOTS; anchor is an ADDRESS.
|
||
let val = if method.is_address() {
|
||
json!(format!("{v:#x}"))
|
||
} else {
|
||
json!(v)
|
||
};
|
||
(dates[b].clone(), val)
|
||
})
|
||
})
|
||
.collect();
|
||
report.push(json!({
|
||
"name": it.name, "tier": it.tier, "method": method.as_str(),
|
||
"builds_located": count, "of_builds": n,
|
||
"first_build": first.map(|f| dates[f].clone()),
|
||
"consistency": (consistency * 100.0).round() / 100.0,
|
||
"history": history,
|
||
}));
|
||
}
|
||
|
||
eprintln!(
|
||
"backfill: {} names over {} corpus builds / {} model builds",
|
||
items.len(),
|
||
anchor_dates.len(),
|
||
model_dates.len()
|
||
);
|
||
eprintln!(
|
||
" graduates: {} DEEP first-class (>=50% coverage, >=0.9 consistent), {} recent-but-consistent, {} flaky, {} not located",
|
||
buckets[0], buckets[1], buckets[2], buckets[3]
|
||
);
|
||
let doc = json!({
|
||
"names": items.len(), "corpus_builds": anchor_dates.len(),
|
||
"model_builds": model_dates.len(), "report": report,
|
||
});
|
||
match out {
|
||
Some(p) => {
|
||
std::fs::write(p, serde_json::to_string(&doc)?)
|
||
.with_context(|| format!("write {}", p.display()))?;
|
||
eprintln!(" -> {}", p.display());
|
||
}
|
||
None => println!("{}", serde_json::to_string_pretty(&doc)?),
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
// ---- the identity check's key form. This guard rejects an address the binary itself says is a
|
||
// different function, and it is keyed by SHORT library name. A caller holding the FILE name made
|
||
// `get` miss, `?` return, and the whole check pass silently — which is indistinguishable from "no
|
||
// contradiction found". These pin both halves: that it fires, and that the wrong form is loud. ----
|
||
fn identity_fixture() -> Identity {
|
||
Identity(HashMap::from([(
|
||
"server".to_string(),
|
||
LibIdentity {
|
||
// Valve's registry says this address is `CLogicRelay::Trigger`, not what we asked about.
|
||
vscript_at: HashMap::from([(0x1000, "Trigger".to_string())]),
|
||
class_size: HashMap::from([("CBaseEntity".to_string(), 0x100)]),
|
||
},
|
||
)]))
|
||
}
|
||
|
||
// mov %rdi,%r13 ; cmpb $0,0x7bc(%r13) ; ret — reaches `this+0x7bc`, past a 0x100-byte class.
|
||
const REACHES_PAST: &[u8] = &[
|
||
0x49, 0x89, 0xFD, 0x41, 0x80, 0xBD, 0xBC, 0x07, 0x00, 0x00, 0x00, 0xC3,
|
||
];
|
||
|
||
#[test]
|
||
fn a_contradiction_is_found_under_the_short_library_name() {
|
||
let img = crate::elf::CodeImage::for_test(0x1000, REACHES_PAST);
|
||
let why = identity_fixture()
|
||
.contradiction("server", &img, 0x1000, "CBaseEntity::SetAbsOrigin")
|
||
.expect("registry disagrees AND the reach exceeds the class — both halves hold");
|
||
assert!(why.contains("Trigger"), "{why}");
|
||
}
|
||
|
||
#[test]
|
||
#[should_panic(expected = "keyed by SHORT library name")]
|
||
fn the_file_name_form_is_loud_rather_than_a_silent_pass() {
|
||
let img = crate::elf::CodeImage::for_test(0x1000, REACHES_PAST);
|
||
// The same contradiction, looked up by the form `images` is keyed by. Before this was asserted it
|
||
// returned `None` — "no contradiction" — on every call.
|
||
identity_fixture().contradiction("libserver.so", &img, 0x1000, "CBaseEntity::SetAbsOrigin");
|
||
}
|
||
|
||
// ---- contribution intake gate (the fork-facing surface) ----
|
||
#[test]
|
||
fn is_iso_date_accepts_only_yyyy_mm_dd() {
|
||
assert!(is_iso_date("2026-07-23"));
|
||
assert!(is_iso_date("2015-01-01"));
|
||
// wrong shape / separators / length / non-digit all rejected — these would otherwise slip past
|
||
// intake and only fail deep in the timeline builder's `src[4]==b'-'` check.
|
||
assert!(!is_iso_date("2026-7-3")); // unpadded
|
||
assert!(!is_iso_date("07/23/2026")); // wrong separator
|
||
assert!(!is_iso_date("2026-07-23T00:00")); // trailing time
|
||
assert!(!is_iso_date("2026_07_23")); // underscores
|
||
assert!(!is_iso_date("not-a-date")); // non-digit in digit slots
|
||
assert!(!is_iso_date("")); // empty
|
||
}
|
||
|
||
// ---- hops identity-compression: an identity alignment is `Identity(len)` IN MEMORY and one integer on
|
||
// the wire; a shift is `Explicit` and a full array. Both forms round-trip, and a pre-compression model
|
||
// (identity stored as a full array) must load to the identical `Identity` Hop. ----
|
||
#[test]
|
||
fn hop_serde_compresses_identity_and_round_trips() {
|
||
let s = |o: Option<usize>| Slot::from_opt(o);
|
||
// identity/empty alignment -> the Identity variant -> a single integer on the wire.
|
||
let ident = Hop::from_slots(vec![s(Some(0)), s(Some(1)), s(Some(2))]);
|
||
assert!(matches!(ident, Hop::Identity(3)));
|
||
assert_eq!(serde_json::to_string(&ident).unwrap(), "3");
|
||
assert!(matches!(Hop::from_slots(vec![]), Hop::Identity(0)));
|
||
assert_eq!(serde_json::to_string(&Hop::default()).unwrap(), "0");
|
||
// any non-identity alignment (a drop, or a reorder) -> Explicit -> the explicit array, nulls for drops.
|
||
let dropped = Hop::from_slots(vec![s(Some(0)), s(None), s(Some(2))]);
|
||
assert!(matches!(dropped, Hop::Explicit(_)));
|
||
assert_eq!(serde_json::to_string(&dropped).unwrap(), "[0,null,2]");
|
||
// every form round-trips (Hop derives PartialEq; Slot has no Debug so assert! not assert_eq!).
|
||
for slots in [
|
||
vec![s(Some(0)), s(Some(1)), s(Some(2))], // identity
|
||
vec![], // empty
|
||
vec![s(Some(0)), s(None), s(Some(2))], // dropped slot
|
||
vec![s(Some(1)), s(Some(0))], // a real reorder = non-identity
|
||
] {
|
||
let h = Hop::from_slots(slots);
|
||
let j = serde_json::to_string(&h).unwrap();
|
||
let back: Hop = serde_json::from_str(&j).unwrap();
|
||
assert!(back == h, "round-trip {j}");
|
||
}
|
||
// backward-compat: an identity alignment written as a full array normalizes to the identical
|
||
// Identity(3) the compact integer produces.
|
||
let from_old_array: Hop = serde_json::from_str("[0,1,2]").unwrap();
|
||
let from_new_int: Hop = serde_json::from_str("3").unwrap();
|
||
assert!(from_old_array == from_new_int);
|
||
assert!(matches!(from_old_array, Hop::Identity(3)));
|
||
}
|
||
|
||
// ---- the CS# text the live oracle re-parses round-trips back to the same locators ----
|
||
// render_monolith_cssharp writes the string; run_live_oracle strips comments and reads it via
|
||
// read_gamedata_str. A drift between writer and reader would silently change what gets validated, so
|
||
// pin that the core (sig) + high_confidence (offset) locators survive the round trip byte-for-byte.
|
||
#[test]
|
||
fn cssharp_render_round_trips_through_read_gamedata_str() {
|
||
use model::{Counts, Entry, MonoEntry, MonoMeta, Monolith, Provenance, Tier};
|
||
let mono = Monolith {
|
||
meta: MonoMeta {
|
||
game_key: "csgo".into(),
|
||
game: "Counter-Strike 2".into(),
|
||
source_build: "test".into(),
|
||
version: "t-1-0".into(),
|
||
counts: Counts {
|
||
core: 1,
|
||
high_confidence: 1,
|
||
experimental: 0,
|
||
unresolved: 0,
|
||
},
|
||
alias_groups: 0,
|
||
aliased_names: 0,
|
||
},
|
||
core: BTreeMap::from([(
|
||
"CBaseEntity::TakeDamage".to_string(),
|
||
MonoEntry {
|
||
locator: Entry::signature("server", "48 8B 05 ? ? ? ?"),
|
||
abi: None,
|
||
provenance: Provenance {
|
||
source: Some("catalogue".into()),
|
||
..Provenance::with_tier(Tier::Core)
|
||
},
|
||
validated: None,
|
||
aliases: Vec::new(),
|
||
},
|
||
)]),
|
||
high_confidence: BTreeMap::from([(
|
||
"CCSPlayerPawn::IsBot".to_string(),
|
||
MonoEntry {
|
||
locator: Entry::offset(42),
|
||
abi: None,
|
||
provenance: Provenance::with_tier(Tier::SelfNamed),
|
||
validated: None,
|
||
aliases: Vec::new(),
|
||
},
|
||
)]),
|
||
experimental: BTreeMap::new(),
|
||
unresolved: BTreeMap::new(),
|
||
};
|
||
let text = render::render_monolith_cssharp(&mono, model::TierSelect::HighConfidence);
|
||
let gd = read_gamedata_str(&text).expect("rendered cssharp must parse back");
|
||
let sig = render::entry_from_value(&gd["CBaseEntity::TakeDamage"]);
|
||
assert_eq!(
|
||
sig.signature.map(|s| s.linux).as_deref(),
|
||
Some("48 8B 05 ? ? ? ?")
|
||
);
|
||
let off = render::entry_from_value(&gd["CCSPlayerPawn::IsBot"]);
|
||
assert_eq!(off.offset, Some(42));
|
||
}
|
||
|
||
// ---- link_aliases groups by LOCATOR IDENTITY, across tiers, and refuses to group a bare slot. The
|
||
// last of those is the one worth pinning: an `offset` with no `class` names no vtable, so bucketing
|
||
// those by slot index would put every unbound "slot 0" in one group and assert 1,080 CS2 names are
|
||
// aliases of each other. ----
|
||
#[test]
|
||
fn link_aliases_groups_by_locator_and_never_by_a_bare_slot() {
|
||
use model::{Entry, MonoEntry, Provenance, Tier};
|
||
let e = |loc: Entry| MonoEntry {
|
||
locator: loc,
|
||
abi: None,
|
||
provenance: Provenance::with_tier(Tier::Core),
|
||
validated: None,
|
||
aliases: Vec::new(),
|
||
};
|
||
let sig = |lib: &str, pat: &str| e(Entry::signature(lib, pat));
|
||
let slot = |class: Option<&str>, n: i64| {
|
||
e(Entry {
|
||
class: class.map(str::to_string),
|
||
..Entry::offset(n)
|
||
})
|
||
};
|
||
|
||
let mut core: BTreeMap<String, MonoEntry> = BTreeMap::from([
|
||
("A::Make".into(), sig("server", "48 8B 05")),
|
||
("Make".into(), sig("server", "48 8B 05")),
|
||
// same pattern, DIFFERENT library — a different function, so not a group.
|
||
("Other::Make".into(), sig("engine2", "48 8B 05")),
|
||
// bare slots: identical index, no class, so nothing links them.
|
||
("IdleState::OnEnter".into(), slot(None, 0)),
|
||
("HideState::OnEnter".into(), slot(None, 0)),
|
||
]);
|
||
let mut high: BTreeMap<String, MonoEntry> = BTreeMap::from([
|
||
// cross-tier: joins the `core` pair above, which is the pairing a reader is least likely to spot.
|
||
("UTIL::Make".into(), sig("server", "48 8B 05")),
|
||
("CFoo::Tick".into(), slot(Some("CFoo"), 12)),
|
||
("CFoo::Update".into(), slot(Some("CFoo"), 12)),
|
||
("CBar::Tick".into(), slot(Some("CBar"), 12)),
|
||
]);
|
||
|
||
let (groups, covered) = link_aliases(&mut core, &mut high);
|
||
assert_eq!((groups, covered), (2, 5)); // the 3-name sig group + the 2-name CFoo group
|
||
|
||
// Cross-tier membership, key-sorted, and never self-referential.
|
||
assert_eq!(core["A::Make"].aliases, ["Make", "UTIL::Make"]);
|
||
assert_eq!(high["UTIL::Make"].aliases, ["A::Make", "Make"]);
|
||
// A shared pattern in another library is a different address.
|
||
assert!(core["Other::Make"].aliases.is_empty());
|
||
// A class-bound slot groups; the same slot on another class does not join it.
|
||
assert_eq!(high["CFoo::Tick"].aliases, ["CFoo::Update"]);
|
||
assert!(high["CBar::Tick"].aliases.is_empty());
|
||
// The whole point: two bare slot-0 entries are NOT claimed to be the same function.
|
||
assert!(core["IdleState::OnEnter"].aliases.is_empty());
|
||
assert!(core["HideState::OnEnter"].aliases.is_empty());
|
||
}
|
||
|
||
// ---- annotate_validation must NOT claim validation it didn't perform ("never lies"). A live-confirmed
|
||
// entry -> Some(true); an entry the oracle KEPT-BUT-COULD-NOT-CHECK (verdict None) -> None (unverified,
|
||
// NOT true); an entry the oracle dropped (absent from the verdict map) -> Some(false). ----
|
||
#[test]
|
||
fn annotate_validation_does_not_overclaim() {
|
||
use model::{Counts, Entry, MonoEntry, MonoMeta, Monolith, Provenance, Tier};
|
||
let entry = |loc: Entry, tier| MonoEntry {
|
||
locator: loc,
|
||
abi: None,
|
||
provenance: Provenance::with_tier(tier),
|
||
validated: None,
|
||
aliases: Vec::new(),
|
||
};
|
||
let mut mono = Monolith {
|
||
meta: MonoMeta {
|
||
game_key: "csgo".into(),
|
||
game: "Counter-Strike 2".into(),
|
||
source_build: "test".into(),
|
||
version: "t-1-0".into(),
|
||
counts: Counts {
|
||
core: 2,
|
||
high_confidence: 1,
|
||
experimental: 0,
|
||
unresolved: 0,
|
||
},
|
||
alias_groups: 0,
|
||
aliased_names: 0,
|
||
},
|
||
core: BTreeMap::from([
|
||
(
|
||
"Live::Fn".to_string(),
|
||
entry(Entry::signature("server", "48 8B"), Tier::Core),
|
||
),
|
||
(
|
||
"Unchecked::Fn".to_string(),
|
||
entry(Entry::signature("vscript", "55 48"), Tier::Core),
|
||
),
|
||
]),
|
||
high_confidence: BTreeMap::from([(
|
||
"Dropped::Fn".to_string(),
|
||
entry(Entry::offset(42), Tier::SelfNamed),
|
||
)]),
|
||
experimental: BTreeMap::new(),
|
||
unresolved: BTreeMap::new(),
|
||
};
|
||
// Live::Fn was live-confirmed; Unchecked::Fn was kept but its lib wasn't mapped (verdict None);
|
||
// Dropped::Fn is ABSENT from the map (the oracle dropped it).
|
||
let verdicts = BTreeMap::from([
|
||
("Live::Fn".to_string(), Some(true)),
|
||
("Unchecked::Fn".to_string(), None),
|
||
]);
|
||
annotate_validation(&mut mono, &verdicts);
|
||
assert_eq!(mono.core["Live::Fn"].validated, Some(true)); // confirmed -> true
|
||
assert_eq!(mono.core["Unchecked::Fn"].validated, None); // kept-unverified -> NOT true, never claimed
|
||
assert_eq!(mono.high_confidence["Dropped::Fn"].validated, Some(false)); // dropped -> false
|
||
}
|
||
}
|