From c458b4cb50ef1ade9b207ba27f16c6ad844de030 Mon Sep 17 00:00:00 2001 From: Kamal Tufekcic Date: Wed, 29 Jul 2026 20:09:21 +0300 Subject: [PATCH] read what the binary says about itself: names, signatures, prototypes; gen v2 --- .forgejo/workflows/ci.yml | 2 +- .forgejo/workflows/contribution.yml | 6 - .forgejo/workflows/derive.yml | 33 +- ATTRIBUTIONS.md | 18 + Cargo.lock | 2 +- README.md | 350 +- crates/source2rosetta-core/Cargo.toml | 2 +- crates/source2rosetta-core/README.md | 77 +- .../src/bin/source2rosetta-gen.rs | 26 +- crates/source2rosetta-core/src/model.rs | 509 +- crates/source2rosetta-core/src/render.rs | 1189 +- fuzz.sh | 12 +- fuzz/Cargo.toml | 21 + fuzz/fuzz_targets/fuzz_concmd.rs | 45 + fuzz/fuzz_targets/fuzz_pulse.rs | 34 + fuzz/fuzz_targets/fuzz_schema.rs | 10 + fuzz/fuzz_targets/fuzz_valvetab.rs | 64 + fuzz/gen_corpus.rs | 8 + fuzz/lsan_suppressions.txt | 2 +- mappings/ehandle-classes.json | 1345 + mappings/prototypes.json | 50538 ++++++++++++++++ src/abi.rs | 83 +- src/concmd.rs | 483 + src/elf.rs | 19 + src/lib.rs | 4 + src/main.rs | 16 +- src/pipeline.rs | 960 +- src/produce.rs | 163 +- src/profile.rs | 39 +- src/prototypes.rs | 911 + src/pulse.rs | 616 + src/rtti.rs | 7 +- src/schema.rs | 417 +- src/valvetab.rs | 544 + 34 files changed, 58363 insertions(+), 192 deletions(-) create mode 100644 fuzz/fuzz_targets/fuzz_concmd.rs create mode 100644 fuzz/fuzz_targets/fuzz_pulse.rs create mode 100644 fuzz/fuzz_targets/fuzz_valvetab.rs create mode 100644 mappings/ehandle-classes.json create mode 100644 mappings/prototypes.json create mode 100644 src/concmd.rs create mode 100644 src/prototypes.rs create mode 100644 src/pulse.rs create mode 100644 src/valvetab.rs diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index f46cdc7..11844c8 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - uses: actions/checkout@v6.0.2 - name: Clear artifacts from any earlier run run: rm -rf fuzz/artifacts - - name: Fuzz — 5 targets x 2 workers x 30s + - name: Fuzz — 8 targets x 2 workers x 30s run: bash fuzz.sh 30 2 - name: Fail on any crash / timeout / OOM run: | diff --git a/.forgejo/workflows/contribution.yml b/.forgejo/workflows/contribution.yml index d55e310..b36ec15 100644 --- a/.forgejo/workflows/contribution.yml +++ b/.forgejo/workflows/contribution.yml @@ -86,9 +86,6 @@ jobs: - name: Produce — validate the contribution live (no model fold; the build is unchanged) run: | - # The model already contains this build (derive folded it), and the forward-only guard needs the - # target to sort strictly after the model's latest. "-" does: it shares the buildid - # prefix and is longer, and the next update's buildid still sorts above it. ln -sfn "$GAME_DIR" "work/$BUILDID-$PATCH" ./target/release/source2rosetta --game "$GAME" produce \ --seed "seed-$GAME.patched.json" \ @@ -100,9 +97,6 @@ jobs: cp "seed-$GAME.patched.json" "dist/seed-$GAME.json" - name: Drop the re-folded model, compress the seed - # A contribution does not change the binary, so the sidecar fold just re-folds a build the model - # already has. Publishing that would append a duplicate row and grow the model on every PR — so the - # model asset is left alone and `-latest` keeps the one the last derive published. run: | rm -f "dist/model-$GAME.json" gzip -6 "dist/seed-$GAME.json" diff --git a/.forgejo/workflows/derive.yml b/.forgejo/workflows/derive.yml index 93e51e8..0c48ada 100644 --- a/.forgejo/workflows/derive.yml +++ b/.forgejo/workflows/derive.yml @@ -23,6 +23,7 @@ jobs: STEAM_APPS: /home/cs2/.steam/SteamApps STEAM_USER: source2rosetta RELEASE_BASE: ${{ github.server_url }}/${{ github.repository }}/releases/download + OVERRIDE_DIR: /home/cs2/rosetta-override steps: - uses: actions/checkout@v6.0.2 @@ -34,9 +35,6 @@ jobs: *) echo "unknown game '$GAME' (expected cs2 or dota2)"; exit 1 ;; esac echo "APPID=$APPID" >> "$GITHUB_ENV" - # No password here: the runner holds a cached refresh token from a one-time interactive login, so - # this is non-interactive. If it ever fails with a login error the token has lapsed — re-run - # `steamcmd +login $STEAM_USER` once on the runner, as the runner user. steamcmd +login "$STEAM_USER" +app_update "$APPID" +quit - name: Resolve the game paths + the new buildid @@ -52,24 +50,29 @@ jobs: - name: Build the deriver run: cargo build --release - - name: Fetch the previous model + seed (the two non-user-facing release artifacts) + - name: Fetch the previous model + seed (or take a manually-placed override) run: | mkdir -p in dist work - curl -fsSL -o in/model.gz "$RELEASE_BASE/$GAME-latest/model-$GAME.json.gz" - gunzip -c in/model.gz > "in/model-$GAME.json" - curl -fsSL -o in/seed.gz "$RELEASE_BASE/$GAME-latest/seed-$GAME.json.gz" - gunzip -c in/seed.gz > "in/seed-$GAME.json" + OVR="$OVERRIDE_DIR/$GAME" + for stem in "model-$GAME.json" "seed-$GAME.json"; do + if [ -f "$OVR/$stem" ]; then + echo "::warning::baseline OVERRIDE in use for $stem — taken from $OVR, not the $GAME-latest release" + cp "$OVR/$stem" "in/$stem" + echo "OVERRIDE_USED=1" >> "$GITHUB_ENV" + else + curl -fsSL -o "in/$stem.gz" "$RELEASE_BASE/$GAME-latest/$stem.gz" + gunzip -c "in/$stem.gz" > "in/$stem" + fi + done - name: Produce — derive + validate-live + typed netvars + fold model N -> N+1 run: | - # --target is a buildid-named SYMLINK to the install, not the install path itself: the deriver labels - # a build by its directory NAME, and the forward-only guard demands each target sort strictly after - # the model's latest build. Passing ".../game" would label every build "game", so the SECOND run - # would be rejected as not-newer. Buildids are monotonic and sort after the corpus's date labels. ln -sfn "$GAME_DIR" "work/$BUILDID" ./target/release/source2rosetta --game "$GAME" produce \ --seed "in/seed-$GAME.json" \ --corpus-model "in/model-$GAME.json" \ + --prototypes mappings/prototypes.json \ + --ehandle-classes mappings/ehandle-classes.json \ --target "work/$BUILDID" \ --game-dir "$GAME_DIR" \ --version "$GAME-$BUILDID-0" \ @@ -104,3 +107,9 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} override: true release-notes: "Rolling ${{ env.GAME }} gamedata — always the newest build (currently ${{ env.BUILDID }}). Stable URL; assets overwritten each update." + + - name: Consume the baseline override + if: env.OVERRIDE_USED != '' + run: | + rm -f "$OVERRIDE_DIR/$GAME"/* + echo "override consumed — subsequent $GAME runs resume from the $GAME-latest release" diff --git a/ATTRIBUTIONS.md b/ATTRIBUTIONS.md index 3decaa5..8128484 100644 --- a/ATTRIBUTIONS.md +++ b/ATTRIBUTIONS.md @@ -52,6 +52,24 @@ The catalogues and dictionaries that give the derived offsets/signatures their n [hazedumper](https://github.com/frk1/hazedumper) (frk1). - **macOS symbol ground-truth** — [dota-2-symbols](https://github.com/a2x/dota-2-symbols) (a2x): symbolicated Dota builds that anchor cross-game name transfer. +- **Declared prototypes** (`mappings/prototypes.json`) — the parameter and return types committed with this + repository are harvested from the macOS symbols above, from `SH_DECL_HOOK` declarations in open-source + mods, and from [modsharp-public](https://github.com/Kxnrl/modsharp-public) (Kxnrl, AGPL-3.0-or-later), + whose gamedata-keyed function-pointer typedefs are the source of most of the declared RETURN types. Only + files ModSharp itself authored are mined; the ten marked "modified from alliedmodders/hl2sdk/tree/cs2" + are denylisted. Also from [CS2Fixes](https://github.com/Source2ZE/CS2Fixes) (Source2ZE, GPL-3.0), whose + detour block declares each hooked engine function as a real function-pointer type; those are joined to + our names by BYTE PATTERN or by a label that already is one of our names, never by a bare method name. + Its vendored `sdk/` (alliedmodders/hl2sdk) is quarantined like every other copy, and its + `serversideclient.h` is excluded on provenance — it carries a `@Wend4r` annotation, i.e. it descends + from a fork this project rejected, and every copy of that header across the ecosystem shares the lineage. + Also from [cs2kz-metamod](https://github.com/KZGlobalTeam/cs2kz-metamod) (KZGlobalTeam, AGPL-3.0), whose + detour block covers the movement surface no other source declares, and from + [plugify-plugin-s2sdk](https://github.com/untrustedmodders/plugify-plugin-s2sdk) (untrustedmodders, + GPL-3.0), whose `addresses` table and hook aliases are full function-pointer types and whose + `CALL_VIRTUAL` sites contribute return types. Both are joined by the same two routes. plugify's + `external/` is excluded entirely: two of its submodules are Wend4r forks, one of them the very + sourcesdk tree rejected above, so only its own `src/` is ever read. - **Dota / Deadlock gamedata** — [McDota](https://github.com/LWSS/McDota) (LWSS), [dota2dumped](https://github.com/ikhsanprasetyo/dota2dumped) & [Dota2Cheat](https://github.com/ikhsanprasetyo/Dota2Cheat) (ikhsanprasetyo), [D2VDump](https://github.com/ModDota/D2VDump) (ModDota), diff --git a/Cargo.lock b/Cargo.lock index a07b904..c3a9cf0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -248,7 +248,7 @@ dependencies = [ [[package]] name = "source2rosetta-core" -version = "0.1.0" +version = "2.0.0" dependencies = [ "anyhow", "clap", diff --git a/README.md b/README.md index 894e748..a00cd5e 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,17 @@ When Valve ships an engine update, every Metamod / CounterStrikeSharp plugin bre Here it takes **about half an hour, with nobody involved.** A timer notices the new build, re-derives the whole surface from the stripped `.so` libraries the dedicated server maps, launches its own vanilla server and *calls the functions* to prove they resolve, then publishes to a fixed URL. No one is paged and nothing is hand-checked — and if any stage fails, the run stops and the previous release stays up. What ships is never a guess. ```sh -# always the newest build -curl -fsSLO https://git.lo.sh/kamal/source2rosetta/releases/download/cs2-latest/gamedata-cs2.json -curl -fsSLO https://git.lo.sh/kamal/source2rosetta/releases/download/cs2-latest/netvars-cs2.json +R=https://git.lo.sh/kamal/source2rosetta/releases/download/cs2-latest # always the newest build +curl -fsSLO $R/gamedata-cs2.json # WHERE functions are — signatures + vtable offsets +curl -fsSLO $R/netvars-cs2.json # field offsets and types +curl -fsSLO $R/abi-cs2.json # HOW to call them — parameter and return types, re-judged per build ``` -The output is framework-neutral; `source2rosetta-gen` renders it into whatever your stack speaks. The deriver behind it is a standalone Rust tool — you only need that if you're self-hosting the pipeline or adding a game. +Also published: `bindings-.json` (the callable surface the binary declares about itself — Pulse +bindings, entity IO, console commands) and `manifest.json` (which build you got). The +[artifacts section](#artifacts-schemas--output-formats) covers all of them. + +The output is framework-neutral; `source2rosetta-gen` renders it into whatever your stack speaks — the gamedata into your framework's locator format, and `abi-.json` into **typed call sites** for the same functions. The deriver behind it is a standalone Rust tool — you only need that if you're self-hosting the pipeline or adding a game. ## Docs @@ -27,10 +32,10 @@ Ballpark from a recent build, on a 16-core desktop. These move build-to-build | | derived functions | typed schema | model | one-time distill | |---|---|---|---|---| -| **CS2** | ~1,150 `core` + ~1,200 `high_confidence`, all live-validated, plus ~4,400 `experimental` name guesses | ~1,900 classes / ~12,300 fields | ~48 MB (a few MB gzipped) | ~15 min | -| **Dota 2** | ~1,900 `core` + ~1,000 `high_confidence`, plus ~6,100 `experimental` | ~2,960 classes / ~17,700 fields | ~570 MB | ~1 hr | +| **CS2** | ~1,125 `core` + ~2,620 `high_confidence`, plus ~4,375 `experimental` name guesses | ~1,900 classes / ~12,300 fields | ~48 MB (a few MB gzipped) | ~15 min | +| **Dota 2** | ~1,930 `core` + ~2,450 `high_confidence`, plus ~5,950 `experimental` | ~2,960 classes / ~17,700 fields | ~570 MB | ~1 hr | -Both derive **0-dropped** — every offset and signature that ships passed live validation. Distilling the model is a one-time cost; after that each build's re-derive is minutes of compute, and the half hour in the headline is the whole loop: notice, update, derive, validate, publish. +A full run live-validates what it ships and reports **0 dropped** on both games — for CS2 that is ~2,610 signatures and ~1,120 vtable offsets checked against a running server. Distilling the model is a one-time cost; after that each build's re-derive is minutes of compute, and the half hour in the headline is the whole loop: notice, update, derive, validate, publish. --- @@ -42,7 +47,7 @@ Each game runs its own loop, independently: 2. On a change it updates the install and runs a single `produce`: derive → live-validate → typed netvars → roll the model forward. 3. It publishes an immutable `--` snapshot, then moves `-latest` onto it. -A CS2 update never rebuilds Dota, and vice versa. Two rules keep it honest: every stage **hard-fails rather than substituting** an older or on-disk input, and every entry in `core` / `high_confidence` is confirmed against the live process before it ships. A failed run publishes nothing and leaves the previous release standing. +A CS2 update never rebuilds Dota, and vice versa. Two rules keep it honest: every stage **hard-fails rather than substituting** an older or on-disk input, and every `core` / `high_confidence` entry that *can* be checked against the live process is checked before it ships. (A handful legitimately cannot — see [`validated` is three-valued](#validated-is-three-valued).) A failed run publishes nothing and leaves the previous release standing. | you want | use | |---|---| @@ -54,49 +59,163 @@ Follow `-latest` to adopt updates as they land, or pin a buildid tag to adopt th --- -## How it works — read → derive → validate → emit +## How it works + +Six stages: **read → derive → locate → measure → validate → emit.** The organising distinction, which everything else hangs off: + +> **Some sources LOCATE a function. Others only DOCUMENT it.** +> A source that pairs a name with an address gives you a locator. A source that pairs a name with +> documentation gives you a catalogue entry and nothing to call. Conflating the two is the single most +> expensive mistake this project has made, and the artifacts keep them apart deliberately. ### 1. Read — the whole server, not just `libserver` -A dedicated server maps roughly 22 shared libraries, and `libserver.so` is only a fraction of the reachable engine surface. source2rosetta reads **all of them** (multilib): `libserver`, `libengine2`, `libtier0`, `libnetworksystem`, `libschemasystem`, and the rest — each locator it emits carries the library it belongs to. ELF parsing is done by hand; `.rela.dyn` relocations are resolved so that `.data.rel.ro` pointer slots (zero on disk) come back as their true as-loaded values. +A dedicated server maps roughly 22 shared libraries, and `libserver.so` is only a fraction of the reachable engine surface. source2rosetta reads **all of them** (multilib) — `libserver`, `libengine2`, `libtier0`, `libnetworksystem`, `libschemasystem`, and the rest — and every locator it emits carries the library it belongs to. -### 2. Derive +ELF parsing is by hand. Three things make a stripped binary readable at all: -**Offsets & netvars — near-deterministic.** Vtable offsets come from **Itanium C++ RTTI**: the type hierarchy and vtable layout are read straight out of the binary, so a method's slot index is a fact, not a guess. Field layouts come from **Valve's own SchemaSystem** reflection tables — the engine emits class/field metadata (name, type, offset) as static data for its own use, and source2rosetta reads it directly. No fingerprints, no guessing. +- **Relocations are resolved.** Every `SHT_RELA` section is processed (matched by type and flags, never by section name), so `.data.rel.ro` pointer slots — zero on disk — come back as their true as-loaded values. Three relocation types are handled; anything else is left alone. +- **`.eh_frame` enumeration** via `PT_GNU_EH_FRAME` recovers function extents where unwind info survives. Valve strips it from the *game* code, so this covers the statically-linked runtime tail and not much else — which is exactly why the next item exists. +- **Candidate entries** = relocation code-pointers (every vtable slot) ∪ decoded near-call targets, unioned with the `.eh_frame` starts. This is the function list everything downstream iterates. -**Non-virtual signatures — located, then verified.** A stripped, non-virtual function has no slot and no symbol, so it has to be *found*. source2rosetta computes a **recompilation-invariant fingerprint** of each catalogued function — CFG shape, call-graph degree, mnemonic histograms, imported-symbol references. These are deliberately **abstracted statistics, never raw bytes**. Locating a function in a new build is **nearest-history under a plain, unweighted L1 distance** over those features, accepted only within a small fixed recompile threshold. To be exact about what this is *not*: there is **no trained model, no machine learning, no learned or weighted metric, no embedding network** — it's a deterministic nearest-neighbour lookup against recent history, and the per-game "model" it reads is a bundle of derived *facts*, not a network. Every located address is then **re-verified independently of the match**: a fresh byte-signature is regenerated at the predicted address and confirmed to be **unique** in the target library and on a **function prologue** — the same check a loader does. A match that doesn't verify isn't shipped. +On top of that sits a whole-binary **cross-reference index**: for every referenced address, the instructions that reference it. That gives the string-anchored locator ("which function uses this string?"), which several later stages depend on. -**ABI-shape guard.** A byte-signature sees a function's *body* drift and re-derives it, but it can't see the *argument list* change while the prologue stays recognisable — the sig still resolves and points at real code, yet a caller using the old prototype passes the wrong registers. source2rosetta recovers each function's observable **SysV-AMD64 ABI shape** (which argument registers are live-in, plus the return class) via a bounded backward-liveness pass and diffs it across builds, flagging exactly those prototype changes and marking struct-by-value (sret) returns that are unsafe to blind-call. +All Source-2 `.so` files link at vaddr 0, so a runtime address is simply `load_base + file_vaddr`. -### 3. Validate — against a live server, not a spec +### 2. Derive from the binary's own reflection -This is what separates source2rosetta from a static dumper. `produce` and `integration-test` **launch their own** vanilla dedicated server for the game (bots on an empty deathmatch for pawn games; a pawn-less game like Dota waits on a `ready_class` proxy) — no Steam, no separate instance, no human. Then, reading the running process through `/proc//mem` (read-only ptrace — no injection, no debugger), it checks against ground truth: +Valve compiles two reflection systems into every module, and both are read directly. + +**RTTI** gives the type hierarchy and vtable layout: which classes exist, what they inherit, and the ordering of every vtable. Where a slot is read *directly* — the fold's offset locators and the experimental band — a slot index is a fact, not a guess. + +**SchemaSystem** gives class → field metadata. Be precise about what is static here: **name and offset are in the file; the TYPE is not.** A field's type pointer is a null placeholder on disk and is populated only at runtime, which is why typed netvars require a live process and why an offline run ships no `netvars-.json` at all. + +### 3. Names Valve ships in the binary — three sources, and only two locate + +Every Source-2 module names some of its own functions. This is ground truth from the shipped binary — not inference, not cross-game transfer — so it outranks every derived name, and it re-derives on every build with no input to maintain and nothing to bootstrap: the source travels with the binary. + +| source | what it pairs | locates? | +|---|---|---| +| **entity-IO datadesc** | the C++ handler name (`InputKill`) with the handler's address | **yes** | +| **console-command registration** | the command name (`bot_add`) with its callback | **yes** | +| **Pulse binding registry** | a qualified `Class::Method` with display name, description, call policy and a full typed signature | **no** | + +**The Pulse registry does not locate anything, and the mistake is instructive.** Its records carry two code pointers, which an early pass folded as locators for a headline `+782` names. They are *descriptor accessors* — every CS2 `libserver` binding measures the same empty `int=0 float=0` footprint, and they disassemble to a lazy-init singleton that returns a static vector. Folding them would have shipped `CBaseEntityAPI::GetAbsOrigin` pointing at a zero-argument accessor: a locator that resolves, passes live validation as executable code, and is still the wrong function. The honest yield from that table is zero locators — and a complete typed API surface, which ships separately as `bindings-.json`. + +**The datadesc handlers are class-qualified** by joining each record's array against the SchemaSystem: a datadesc array also carries field descriptors, and a `(member, offset)` pair is something the schema states from an entirely different table, so the class whose schema satisfies every pair in the array owns it. 653 of 715 CS2 handlers qualify this way. This is what makes `InputEnable` — a distinct handler on 48 classes — nameable at all; unqualified names that resolve to several addresses are still **dropped, not guessed**. Note for consumers: qualification **renamed 343 shipped keys** (`InputActivateSkybox` → `CAmbientGeneric::InputActivateSkybox`). + +**Console commands are read from the registration call, not a registry walk.** CS2 registers through a handle-based `ConCommandRef` whose registry lives in tier0, so there is no static `ConCommand` object to scan for and nothing in the file points at a command name. The registration *call* still does — an ordinary call from a static initializer whose name, handler, description and flags are all constants in the instruction stream. source2rosetta tracks what each argument register provably holds and reads the vector at every call: **784 commands across 20 of the 22 libraries on CS2, 855 on Dota** (the two misses are libraries whose registrar the shape test does not find). The registrar is identified by what it *does* — it opens by writing the invalid-handle sentinel into its `ConCommandRef` — never by address and never by ranking call sites, so a library with three commands is as readable as one with three hundred. + +Because name and handler come from one instruction sequence, Valve's own published command dump checks the result: **746 of 746 descriptions agree exactly, with no disagreements**, 742 of 780 distinct names appear in that dump, and 12 flag bits are matched to the flag names it prints. Three further flag bits occur and are deliberately left unnamed. + +**Typed Pulse signatures, read offline.** A binding's record does not carry its signature; the accessors beside it return the parameter and return vectors, which are built at runtime and therefore zero in the file. The *code* that builds them is not, and every field is written to a fixed address from a `lea` or an immediate — so constant-propagating through the initializer recovers the full typed signature with no running process: **999/999 CS2 records and 859/859 Dota**, with parameter names, `PulseValueType_t` types, and the schema type a value refers to where the binding names one. Cross-checked against Valve's published metadata dump: **435 of 437** CS2 server bindings agree exactly, **293 of 294** on Dota (every difference a global-event binding, where Valve documents the same vector as an out-param). + +### 4. Locate across builds — for everything Valve doesn't name + +A stripped non-virtual function has no slot and no symbol, so it has to be *found*. + +**The fingerprint** is a recompilation-invariant statistical description of a function: bounded-CFG structural counts, an 18-bucket mnemonic-class histogram, a 32-bucket sketch of the printable `.rodata` strings it references, and one hop of call-graph context (distinct callees plus an aggregate of each callee's own instructions, calls and branches) — **63 dimensions**, deliberately abstracted statistics and never raw bytes. + +To be exact about what this is *not*: **no trained model, no machine learning, no learned or weighted metric, no embedding network.** Matching is nearest-history under a plain unweighted **L1** distance, accepted within a fixed threshold. The per-game "model" is a bundle of derived *facts* — vtable-alignment hops, reference-fingerprint windows, ABI-shape consensus, slot timelines — not a network. + +**A signature resolves by one of three outcomes, and the artifact says which:** + +1. **Strict** — a fingerprint match inside the threshold. +2. **Lenient** — no fingerprint check, but two or more distinct era-signatures vote for the same address. +3. **Unverified fallback** — a single candidate that every check rejected, shipped anyway and marked **`catalogue-unverified`**. This is **0.4% of CS2's core and 69.8% of Dota's**, so on Dota it is the common case, not an edge case. Read the marker. + +Whatever the route, the shipped byte-signature is regenerated at the resolved address and confirmed **unique in its library**. That is a *usability* check, not corroboration of identity — the pattern is generated *from* that address, so a wrong address yields a signature that is unique and equally wrong. Identity comes from the match, the vote, or live validation; uniqueness only ensures a loader can find it. + +**Catalogue vtable offsets are chained, not read.** For a named method, the slot in a *new* build is derived: dated slots are chained forward through per-build-pair vtable alignments that are themselves fingerprint-scored, then a recency-weighted vote emits a slot only above **80% confidence**. RTTI supplies the vtable and the ordering; which slot a named method occupies is an inference with a stated bar. + +### 5. Measure the ABI — the guard a byte-signature can't provide + +A signature sees a function's *body* drift and re-derives it. It cannot see the *argument list* change while the prologue stays recognisable — the signature still resolves and points at real code, yet a caller using the old prototype loads the wrong registers. + +So each function's observable **SysV-AMD64 shape** (which argument registers are live-in, plus the return class) is recovered by a bounded backward-liveness pass and diffed across builds, flagging exactly those prototype changes and marking struct-by-value returns that are unsafe to blind-call. + +The footprint is a deliberate **lower bound** — a callee that ignores an argument reads fewer registers than it is passed — and that is checked rather than asserted. Valve's entity-IO datadesc declares hundreds of independent handlers to one fixed prototype, and every one measures within it: **CS2 715/715, Dota 624/624**, on every derive. + +Types cannot be recovered from a stripped binary, so `abi-.json` joins *declared* prototypes to that measurement and judges each one. See [the manifest](#abi-gamejson--declared-prototypes-judged-against-this-build). + +### 6. Validate — against a live server, not a spec + +This is what separates source2rosetta from a static dumper. `produce` and `integration-test` **launch their own** vanilla dedicated server (bots on an empty deathmatch for pawn games; a pawn-less game like Dota waits on a `ready_class` proxy) — no Steam, no separate instance, no human. + +Two access paths, and they differ: + +- **Reading** is `/proc//mem` — no attach, no stop, no injection. +- **Calling** is a real debugger attach: `PTRACE_ATTACH`, save registers, write a scratch frame, run, restore. No injected *code*, but the process is stopped and its registers are written. + +What gets checked: - every **offset** lands on a real vtable slot, and every **signature** on live executable code; -- a gamedata function is **actually called** via ptrace to prove it's the semantically right function, not a plausible byte-match (pawn games); +- a gamedata function is **actually called** to prove it is the semantically right function, not a plausible byte-match (pawn games); - derived probes are **fuzzed across changing game state** for many iterations; -- field **types** are read from the live process for the typed netvars — and fields that are non-null live but zero on disk (e.g. `m_pSchemaBinding`) confirm the reader is seeing real live state, not stale disk bytes. +- field **types** are read live for the typed netvars — and fields that are non-null live but zero on disk confirm the reader is seeing real runtime state, not stale disk bytes; +- a **hooked** function (a mod detoured it) is detected by a byte diff at a uniquely-resolved prologue and reported rather than failed. -The contract is blunt: **"degrades or stops loudly, never lies."** An entry live validation confidently rejects is dropped, not shipped under a banner claiming it resolves; if the oracle can't run, it fails loudly rather than emit an unverified result. +The contract is blunt: **"degrades or stops loudly, never lies."** An entry live validation confidently rejects is dropped, not shipped under a banner claiming it resolves; if the oracle cannot run, the run fails rather than emit an unverified result. -### 4. Confidence tiers — nothing vanishes silently - -The output is a per-game **monolith** in which every catalogue entry is accounted for, sorted into four tiers: +### 7. Confidence tiers — nothing vanishes silently | tier | meaning | |---|---| -| `core` | derived and, in a full run, **live-validated** — the load-bearing gamedata | -| `high_confidence` | corroborated names folded in as verified offsets/sigs (dictionary-exact or macOS ground-truth transfer) | -| `experimental` | the least-filtered band — every graded name guess, each with a **resolvable locator** but an **unverified name** | -| `unresolved` | catalogued but not confidently produced this build, with a reason (`sig-drifted`, `offset-low-conf`, …) and no locator | +| `core` | derived and, in a full run, live-validated — the load-bearing gamedata | +| `high_confidence` | names folded in as verified offsets/sigs — Valve's own in-binary sources (`valve-table` provenance, ground truth) first, then macOS ground-truth transfer, dictionary-exact, and gated extrapolation | +| `experimental` | the least-filtered band — every graded name guess, each with a **resolvable locator** but an **unverified name**. **Never live-validated.** | +| `unresolved` | catalogued but not confidently produced this build, with a closed-vocabulary reason (`sig-drifted`, `offset-low-conf`, `unresolved`, `abi-drift`) and no locator | A function that couldn't be derived this build shows up as `unresolved` with a reason — it never just disappears. -### 5. Emit +**Naming is not derivation.** AI-assisted name extrapolation exists but is producer-side dev tooling, not part of the shipped deriver; every proposed name is gated on self-naming or ground-truth corroboration and then live-validated. A wrong *name* mislabels a real slot; it never touches the **offset**, which comes from RTTI. -The canonical model and every format emitter live in `crates/source2rosetta-core`; the deriver writes format-neutral JSON and the tiny `source2rosetta-gen` binary renders it into any framework's shape (see [Artifacts & formats](#artifacts-schemas--output-formats)). +--- -Two things to keep straight. **Multi-game:** the ELF / RTTI / SysV / SchemaSystem machinery is engine-generic; game-specific knobs (library set, launch spec, schema-probe classes, pawn anchor) live on a `profile::GameProfile`, and `--game ` selects one — adding a game is a `const` plus a clap arm. **Naming is not derivation:** AI-assisted name extrapolation exists but is producer-side dev tooling, not part of the shipped deriver, and every proposed name is gated on self-naming or ground-truth corroboration and then live-validated. A wrong *name* only mislabels a real slot; it never touches the **offset**, which comes from RTTI. +## What it guarantees — and what it refuses to ship + +The tool's strongest claim is not what it produces but what it declines to produce. + +### Release floors + +A `produce` run **aborts rather than publish** a collapsed artifact. Each surface is gated separately, because each is matched by its own record shape — a Valve reshape that breaks one leaves the others intact, and a summed floor would stay satisfied while a whole surface silently vanished: + +| surface | floored on | why it needs its own | +|---|---|---| +| Pulse bindings | registry record count | the registry can move independently of the typed signatures | +| typed Pulse signatures | recovered-signature count | the registry can read perfectly while the descriptor layout moves — every binding would ship signature-less | +| entity-IO records | inputs + outputs | | +| entity classnames | factory-record count | | +| console commands | recovered-command count | the registrar is found by SHAPE, so a reworked constructor yields **zero** commands rather than wrong ones — correct, and invisible without this | +| typed netvars | fraction of fields typed | a runtime type-layout reshape resolves every field "untyped" and would otherwise ship a typeless schema at exit 0 | +| schema enums | enum count | read by shape like the class table, so a reshape yields zero rather than wrong | +| live validation | pass rate, above a minimum sample | | + +### `validated` is three-valued + +`true` / `false` / `null`, and `null` is not a synonym for failure. Three cases ship `null` legitimately: the library is not mapped in the vanilla server, the class is not a vtable class (an engine special or a carried member offset), or the entry has no locator to check. An **offline run ships the whole monolith `null`** — absence of validation, not failed validation. Treat `null` as "not checked here", never as "checked and passed". + +### The standing oracles + +A dozen free, mostly two-sided checks run on **every** derive and are reported. Two-sided means the two halves are read from different places by readers that don't know about each other, so agreement is evidence and disagreement is a defect: + +- **entity-IO ABI** — hundreds of independent handlers against one declared prototype: CS2 715/715, Dota 624/624. +- **console-command ABI** — the callback *form* comes from the registration site, the *arity* from a liveness pass over a different function body: CS2 784/784, Dota 855/855. +- **console-command distinctness** — 784 commands → 783 distinct handlers (855 → 854 on Dota). Catches a layout change that would start returning a shared dispatch thunk: still resolving, still validating, still wrong. +- **Pulse receiver cross-check** — the registry's policy flag vs the independently recovered parameter list: 999/999 and 859/859 registrations. +- **entity-output ↔ schema join** — 226/226 CS2, 186/187 Dota. +- **EHANDLE class grouping** — Valve's naming vs the binary's destructor addresses: 0 of 44 CS2 / 41 Dota groups carry two classes. +- **Pulse element stride** — derived by consensus per image, unanimous across six libraries in both games. +- **live schema oracle** — offline layout vs the running process: 852/852 CS2, 1,912/1,912 Dota. +- **field-gap size calibration**, the semantic call sweep, and a 500-iteration live fuzz. + +One of these found a real defect on its first run: 34 of 715 handlers measured float arguments a `void(ptr, ref)` cannot have, which traced to the ABI reader treating a `call` as fall-through so a callee's *return* propagated backwards as a phantom argument. + +### What is NOT gated + +Stated because "we check things" is worthless without a boundary. There is **no** floor on the validate-live drop rate, the live-fuzz fault rate, the schema class count, or the RTTI class / base-graph size. A regression in any of those is reported, not refused. + +**One oracle is currently reporting.** The Pulse registry is keyed by qualified name, so a binding registered by several modules keeps one row, and every duplicate is compared against the row already present. On the current builds **CS2 flags 331 of 419 repeat registrations and Dota 271 of 359** as disagreeing — so for those names `bindings-.json` carries one module's account of the signature, not a merged one. It is flagged on every run and is not yet resolved; if you consume `params`/`returns` for a multiply-registered binding, know that. --- @@ -113,21 +232,23 @@ cargo build --release # → ./target/release/source2roset cargo build --release -p source2rosetta-core # → ./target/release/source2rosetta-gen ``` +**Prerequisites.** Stable Rust for both binaries. The live half additionally needs a game install and **ptrace permission** (same user with `yama/ptrace_scope=0`, or `CAP_SYS_PTRACE`). The CI fuzz gate needs nightly Rust, `cargo-fuzz` and GNU `parallel`. The release runner needs `steamcmd`, `jq` and `python3`. Dependencies are deliberately few — ELF parsing, disassembly and the parallel primitive are in-tree rather than pulled in. + `--game ` is a global flag (default `cs2`), given before the subcommand: `source2rosetta --game dota2 produce …`. | command | one line | |---|---| | `produce` | The whole per-game build in one command: derive → fold → (with `--game-dir`) validate-live + typed netvars → roll the model forward, into `--out-dir`. **`--game-dir` present = full live-validated build; absent = fast offline build (gamedata + model only). That flag is the entire offline/full switch.** | | `corpus-model` | Distill a corpus of past builds into one shippable model (vtable-alignment hops, reference fingerprints, slot timelines), so future derivation needs only the model + the target binary, not the corpus. | -| `fold-model` | Roll an existing model forward by ONE build (`model N + build → N+1`), reading only the model and that one binary — equal to a full re-distill. The production update path (also a sidecar inside `produce`). | +| `fold-model` | Roll an existing model forward by ONE build (`model N + build → N+1`), reading only the model and that one binary. The production update path (also a sidecar inside `produce`). | | `integration-test` | Stand-alone CI live oracle: launch a vanilla server, populate it, and verify derived gamedata against it — schema oracle, a semantic ptrace CALL on a live pawn, and (with `--gamedata`) a full validate-live plus optional live fuzzing. | | `backfill` | Give an extrapolated name a real cross-build timeline — resolve its string anchor in every corpus build, or chain a vtable slot through the model — and report history depth + consistency (how a guess graduates to first-class). | -| `classify-change` | `--prev`/`--new` → `skip` / `normal` / `shift` + the exact % of function bodies that changed, comparing with position-dependent bytes masked so a pure layout shift reads as unchanged. Decides whether a build even warrants a re-derive. | -| `filter-corpus` | Collapse runs of code-identical builds to one representative, label each transition `normal`/`shift`, and segment the timeline into toolchain eras. Writes the selection manifest the distill reads. | +| `classify-change` | `--prev`/`--new` → `skip` / `normal` / `shift` + the exact % of function bodies that changed, comparing with position-dependent bytes masked so a pure layout shift reads as unchanged. An **operator primitive** — nothing in the shipped pipeline invokes it; the poller dispatches a derive on any buildid change. | +| `filter-corpus` | Collapse runs of code-identical builds to one representative, label each transition `normal`/`shift`, and segment the timeline into toolchain eras. Writes an **advisory** selection manifest; the distill does not read it (see [corpus curation](#getting-the-corpus-only-to-bootstrap-a-model)). | ### Quickstart -Releases live on the **[releases page](https://git.lo.sh/kamal/source2rosetta/releases)**. For the offline path you need one file from there — the model (`model-.json`) — plus the derive inputs, which ship in this repo under `mappings/`. Put the downloaded model wherever you like; the examples assume it's in the working directory. +Releases live on the **[releases page](https://git.lo.sh/kamal/source2rosetta/releases)**. For the offline path you need one file from there — the model (`model-.json`) — plus the derive inputs, which ship in this repo under `mappings/`. ```sh # OFFLINE — derive gamedata + roll the model forward. No server, fully deterministic. @@ -147,18 +268,31 @@ Releases live on the **[releases page](https://git.lo.sh/kamal/source2rosetta/re --out-dir out ``` -- `--target ` (required) — the build **directory** to derive from; its libraries are searched by name, so pass the directory, not a bare `.so`. -- `--seed ` — one file bundling every derive input (catalogue + optional naming/offset/sig sections). The loose equivalent is `--catalogue ` plus the optional `--promotable` / `--candidates` / `--full-names` / `--extra-offsets` / `--extra-sigs`, all defaulting to empty — **so a brand-new game needs only a catalogue to start deriving.** -- Corpus signal — exactly one of `--corpus-model ` (the normal path: forward-derive from the model + the target binary, and roll the model N→N+1 as a sidecar) or `--corpus ` (fingerprint the raw build binaries on the fly). +- `--target ` — the build **directory** to derive from; `produce` requires a directory and its libraries are searched by name. (Other subcommands accept a bare `.so` as well, which is how `classify-change --prev` is used.) +- `--game-dir ` — must be the **`game/` subtree** of the install, the same directory layout the dedicated server is launched from. +- `--seed ` — one file bundling every derive input. The loose equivalent is `--catalogue ` plus the optional `--promotable` / `--candidates` / `--full-names` / `--extra-offsets` / `--extra-sigs`, all defaulting to empty — **so a brand-new game needs only a catalogue to start deriving.** +- Corpus signal — exactly one of `--corpus-model ` (the normal path: forward-derive from the model + target binary, rolling the model N→N+1 as a sidecar) or `--corpus ` (fingerprint raw build binaries on the fly). + +Model-based derives are **forward-only**: the model describes history up to its newest build, so pointing one at an *older* target is not supported. + +The launched server writes its log to `TMPDIR/-produce.log`, one file per game, and binds a **fixed port** — so two games cannot be produced concurrently on one host without changing it. --- ## Fork it & distill your own model -Nothing is hosted — fork it, `cargo build --release`, and point it at a build on disk. Two ways to run, depending on whether you already have a model: +Nothing is hosted — fork it, `cargo build --release`, and point it at a build on disk. -- **Have a model** (downloaded from releases, or distilled): `--corpus-model model-.json` forward-derives from the model plus *only the target binary* — no corpus on disk. This is the normal path. -- **No model yet:** distill one from a corpus of past builds. "Distilling" is what this project means by "training" — there's no ML (see [Derive](#2-derive) above); the model is the bundle of facts already described: vtable-alignment hops, reference-fingerprint windows, ABI-shape consensus, slot timelines. +### What a fork inherits, and what it must supply + +Everything engine-generic is inherited: ELF/RTTI/SchemaSystem/SysV reading, fingerprinting, the model machinery, live validation, the emitters. Two things a fork must produce for itself: + +- **A catalogue** — the one required input. It is the list of functions you want gamedata *for*: each entry a name plus whatever historical evidence exists (dated vtable slots, per-era signatures, string anchors). Everything else in the seed is optional and defaults to empty. Without a catalogue the tool has nothing to look for. +- **A model**, distilled from a corpus of past builds — or downloaded from a release if you're forking this project's games. + +The **seed bundle** collapses the loose inputs into one file with sections for catalogue, promotable names, candidates, full names, extra offsets, extra sigs, and contributions. `mappings/naming/` is a large frozen input with no in-repo producer — it is data, not something a build regenerates. + +### Distilling ```sh # Distill a corpus into a model (streaming, bounded RAM even over Dota's ~1k builds). @@ -168,11 +302,17 @@ Nothing is hosted — fork it, `cargo build --release`, and point it at a build --out model-cs2.json ``` -`--class-scope` (default `clean` — every real game class, enough for any modding offset to derive model-only) picks which classes get slot hops. **Whatever scope you distill with, `fold-model` and `produce`'s sidecar fold must use the same one.** +`--class-scope` picks which classes get vtable-slot hops — the timelines that let a consumer derive an offset from the model alone: + +- `clean` (default) — every real game class, dropping template instantiations, protobuf message shapes and NetworkVar chainers, whose hops nobody derives an offset from. +- `all` — those too. +- `catalogue` — only what the catalogue names. The catalogue's own classes are always included regardless of scope. + +**Whatever scope you distill with, `fold-model` and `produce`'s sidecar fold must use the same one.** The model records its scope and the fold asserts on it, so a mismatch fails loudly — but CI does not pass the flag, so a non-default scope requires a workflow change too. ### Keeping a model fresh — the incremental fold -Once a model exists you never need the corpus again. `fold-model` rolls it forward one build, reading only the model plus the single new binary — identical to a full re-distill: +Once a model exists you never need the corpus again. `fold-model` rolls it forward one build, reading only the model plus the single new binary: ```sh ./target/release/source2rosetta --game cs2 fold-model \ @@ -182,36 +322,44 @@ Once a model exists you never need the corpus again. `fold-model` rolls it forwa --out model-cs2.next.json ``` -`produce --corpus-model` runs exactly this fold as a sidecar, so a full build both derives *and* advances the model in one command. (`--class-scope` must match the model's.) +`produce --corpus-model` runs exactly this fold as a sidecar, so a full build both derives *and* advances the model in one command. + +The fold equals a full re-distill over the same builds under **three** conditions: the same `--class-scope` (asserted), the same catalogue the model was distilled from (**not** checked — production always folds with the distill's catalogue, but a newcomer gets only a warning), and no class re-appearing across the model's latest-build boundary. In that last case a class the model has never seen is back-filled with absent history — reduced coverage for that name until the next full re-distill, never a wrong offset. ### Getting the corpus (only to bootstrap a model) A corpus is a directory of past builds, one subdirectory of `.so` files per build (`corpus/binaries/