From 55f4087458f4155fa54218061a0e896f1e5ca390 Mon Sep 17 00:00:00 2001 From: Jacob Babor Date: Fri, 26 Jun 2026 21:07:55 -0500 Subject: [PATCH] Document VAND GPS search graph --- docs/compaction-handoff.md | 60 +++++++-- docs/red4ext-logging-shim.md | 8 ++ docs/traffic-system-debrief.md | 76 ++++++++++- tools/analyze_vand_navigation.py | 213 +++++++++++++++++++++++++++++++ 4 files changed, 343 insertions(+), 14 deletions(-) create mode 100644 tools/analyze_vand_navigation.py diff --git a/docs/compaction-handoff.md b/docs/compaction-handoff.md index 6c63540..aedf4c5 100644 --- a/docs/compaction-handoff.md +++ b/docs/compaction-handoff.md @@ -1,14 +1,56 @@ # Compaction Handoff -Current date/time context: 2026-06-20/21, local timezone America/Chicago. +Current date/time context: 2026-06-26, local timezone America/Chicago. + +## 2026-06-26 Static Update + +- Installed RED4ext state is safe: the game plugin directory contains + `EdgeWeightGPS.dll.disabled`, not an active `EdgeWeightGPS.dll`. +- Latest committed safe source before this update: `d68a874 Disable unstable + traffic scorer hook`. +- Current source has `kEnableGpsTrafficEdgeWeightPatch = false` and + `kEnableGpsLocalSearchHooks = false`. +- The real route search is confirmed at `0x44f054`, called from + `0x44cc7c`. It is an A*/Dijkstra-style loop: validate packed handles, push + start state, pop best open-list state, traverse adjacent handles, call + provider filter slot `+0x08`, call provider edge-cost slot `+0x10`, update + predecessor/cost, then materialize handles through `0x44eb68`. +- Both base and filtered cost-provider vtables point slot `+0x10` at + `0x44f838`. The provider vtable RVAs are still `0x2ae6120` (base) and + `0x2ae60f8` (filtered). +- `0x44f838` computes geometric edge distance and multiplies by + `provider[0x08 + ((neighborPoint[0x13] & 0x3f) * 4)]`. +- Base provider constructor `0x451158` fills all 64 class multipliers with + `1.0`. Filtered provider constructor `0x44b9b4` changes the vtable and + `0x44bda4` copies route masks/special zones into the provider, but ordinary + GPS route edge cost is still effectively geometric distance. +- `0x44f7bc` is the heuristic helper, not edge cost. It returns distance to the + target scaled by the constant at `0x1431ef3a0` unless the special route-shape + helper supplies an adjusted distance. +- `0x44eb68` only walks predecessor links and writes packed handles. The richer + final route records are produced afterward by `0x44cc7c`/query result code. +- The packed handles resolve into `VAND` navigation blobs from + `base\worlds\03_night_city\_compiled\default\navigation_*.streamingsector`, + not directly into `all.traffic_persistent`. Final route records can still be + joined to `worldTrafficLanePersistent.nodeRefHash`, but that is downstream of + the search. +- Added `tools/analyze_vand_navigation.py` to decode VAND blobs from extracted + streamingsector JSON. On the three extracted samples: + `files=3 blobs=300 points=5059 coords=5046`, all masks were `0x0003`, and + dominant classes were `1`, `4`, `5`, `3`. +- Practical implication: traffic `maxSpeed` and `all.lane_connections` + probabilities are not the edge-cost surface. The next viable patch should + either hook `0x44f838` and classify edges spatially against precomputed + highway/arterial corridors, or patch the navigation VAND graph/classes after + extracting the relevant streamingsector resources. ## Current Installed State -- Latest installed probe source: resolver-window probe. Use `git log -1 - --oneline` for the exact checkpoint. -- Installed RED4ext plugin: - `Cyberpunk 2077/red4ext/plugins/EdgeWeightGPS/EdgeWeightGPS.dll`. -- Installed build is read-only. It should not alter route behavior. +- Installed RED4ext plugin is disabled: + `Cyberpunk 2077/red4ext/plugins/EdgeWeightGPS/EdgeWeightGPS.dll.disabled`. +- There is no active `EdgeWeightGPS.dll` in the plugin directory. +- Source defaults are currently safe/read-only. They should not alter route + behavior unless rebuilt with hook toggles changed. - Latest archived test log: `logs/EdgeWeightGPS_resolve_handle_probe_4650.log`. - Game is cold as of the latest user message. @@ -34,10 +76,12 @@ In `red4ext/EdgeWeightGPS/src/Main.cpp`: - `kEnableGpsQueryLifecycleHooks = true` - `kEnableGpsRouteJobLifecycleHooks = false` - `kEnableGpsRouteJobStartTrace = false` -- `kEnableGpsLocalSearchHooks = true` +- `kEnableGpsLocalSearchHooks = false` - `kEnableRouteUiTraceHooks = false` -Active hooks: +If rebuilt with the current defaults, active hooks are query/async lifecycle +only. The local search/cost hooks below require `kEnableGpsLocalSearchHooks = +true`: - query lifecycle: - `GPSQuerySubmit 0x70a42c` diff --git a/docs/red4ext-logging-shim.md b/docs/red4ext-logging-shim.md index 82b2889..7af23c0 100644 --- a/docs/red4ext-logging-shim.md +++ b/docs/red4ext-logging-shim.md @@ -303,6 +303,14 @@ Latest full-route join: route-record low-byte classes are `{0,1,2,3,4,5,7,8,9}`, and route-record classes 7/8/9 are highway-heavy. These are different domains, which explains why provider point-class tuning did not map cleanly to highway preference. +- Later static analysis tied the search-side segment/point records to `VAND` + blobs in compiled navigation streamingsector resources, not directly to + `all.traffic_persistent`. The traffic lane hash join happens downstream in + the result/materialization path. +- `tools/analyze_vand_navigation.py` decodes those `VAND` blobs from extracted + streamingsector JSON. In the three extracted samples, all masks were `0x0003` + and the dominant point classes were `1`, `4`, `5`, and `3`, matching the live + edge-cost trace. Current static producer lead: diff --git a/docs/traffic-system-debrief.md b/docs/traffic-system-debrief.md index c0ccb0a..4682f81 100644 --- a/docs/traffic-system-debrief.md +++ b/docs/traffic-system-debrief.md @@ -731,6 +731,60 @@ from `all.traffic_persistent`, which confirms that the displayed/player GPS path is backed by the traffic lane graph even though `maxSpeed` and connection probability patches did not affect route choice. +Later static work found the actual graph-search loop upstream of those result +records: + +```text +route producer 0x44cc7c +packed-handle resolver 0x44e1a8 +path search loop 0x44f054 +path materializer 0x44eb68 +heuristic distance helper 0x44f7bc +edge-cost callback 0x44f838 +base provider filter 0x450b08 +filtered provider filter 0x44ff68 +``` + +`0x44f054` is an A*/Dijkstra-style open-list search. It validates packed start +and target handles, pushes the start node, repeatedly pops the lowest-priority +state, checks the target handle, traverses adjacent handles from the current +navigation segment, calls a provider filter through vtable slot `+0x08`, calls +edge cost through vtable slot `+0x10`, and records the predecessor/cost when a +neighbor improves. + +Both the base and filtered cost providers use `0x44f838` for slot `+0x10`. +That function computes geometric distance between the current and neighbor +points and multiplies it by: + +```text +provider[0x08 + ((neighborPoint[0x13] & 0x3f) * 4)] +``` + +The base provider constructor at `0x451158` initializes all 64 class +multipliers to `1.0`. The filtered provider constructor at `0x44b9b4` reuses the +same edge-cost function and mainly adds route masks/special zones. For normal +map routes, this means the live GPS edge cost is effectively geometric distance +over a compact navigation graph, not `lane.length / lane.maxSpeed`. + +The packed handles resolved by `0x44e1a8` point into `VAND` navigation blobs +loaded from `base\worlds\03_night_city\_compiled\default\navigation_*.streamingsector`. +The 0x14-byte point record has: + +- `+0x00`: adjacency/index field used by the search loop +- `+0x10`: 16-bit provider mask +- `+0x12`: small point metadata byte +- `+0x13`: packed route point class/flags; low six bits are the class used by + `0x44f838` + +The helper `tools/analyze_vand_navigation.py` decodes these blobs from +streamingsector JSON. In the three extracted navigation samples, all blobs were +version 8, all point masks were `0x0003`, and the dominant classes were +`1`, `4`, `5`, and `3`. That matches the runtime edge-cost logs. This explains +why the earlier `all.traffic_persistent` patches changed vehicle simulation but +did not steer player GPS: the route chooser is walking the baked navigation +`VAND` graph, then later materializing/copying results that can be joined to +traffic lane hashes. + Static disassembly found a small traffic candidate scorer at `0x8d46cc`: ```text @@ -779,12 +833,17 @@ RED4ext plugin was disabled. The hook is therefore disabled by default in source (`kEnableGpsTrafficEdgeWeightPatch = false`) and the local installed DLL is renamed to `EdgeWeightGPS.dll.disabled`. -The main takeaway is still valuable: traffic lane flags are available in native -route-side code, and `0x8d46cc` is an excellent landmark. It is not yet a safe -playable patch point. The next native work should avoid a full detour of this -hot scorer and instead look for either: +The main takeaway is still valuable, but narrower than first suspected: +traffic lane flags are available in native route-side endpoint/candidate code, +and `0x8d46cc` is an excellent landmark. It is not the city-wide graph-cost +function. The safer cost target is `0x44f838`, or a data patch against the +navigation `VAND` blobs it consumes. The next native work should avoid a full +detour of `0x8d46cc` and instead look for either: -- the broader search/expansion loop that feeds candidates into this selector +- a `0x44f838` edge-cost hook that classifies graph edges by spatial proximity + to highway/arterial corridors +- a navigation streamingsector patch that marks/weights selected `VAND` graph + points or edges - an inline patch around the vanilla constants/branches if the selector proves sufficient - a safer result-stage rewrite only if upstream weighting cannot be patched @@ -798,7 +857,12 @@ Better versions of this mod could: `0x8d25xx` to `0x8d49xx` cluster - replace the crashing `0x8d46cc` detour with a narrower inline patch or a safer caller-side hook -- locate any native or baked edge-cost data not exposed in RTTI +- decode enough of the `VAND` graph to join navigation points to traffic lane + polygons and highway flags +- extract the full Night City navigation streamingsector set into workspace + storage and summarize it with `tools/analyze_vand_navigation.py` +- implement a bounded `0x44f838` edge-cost hook or `VAND` data patch using that + join, rather than traffic `maxSpeed` - inspect `GPSOnly` connector lanes if a later native trace proves they are consumed by the player GPS - build district-specific presets if a real data-cost surface is found diff --git a/tools/analyze_vand_navigation.py b/tools/analyze_vand_navigation.py new file mode 100644 index 0000000..01e3dbc --- /dev/null +++ b/tools/analyze_vand_navigation.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Summarize VAND navigation-graph blobs from streamingsector JSON files.""" + +from __future__ import annotations + +import argparse +import base64 +import collections +import dataclasses +import json +import math +import struct +from pathlib import Path +from typing import Any, Iterator + + +HEADER_SIZE = 0x64 +POINT_SIZE = 0x14 +COORD_SIZE = 0x0C + + +@dataclasses.dataclass +class VandBlob: + path: Path + ordinal: int + size: int + version: int + tile_x: int + tile_y: int + point_count: int + coord_count: int + aux_count: int + point_count_2: int + aux_count_2: int + point_count_3: int + class_counts: collections.Counter[int] + raw_class_counts: collections.Counter[int] + mask_counts: collections.Counter[int] + bounds: tuple[float, float, float, float, float, float] | None + + +def walk_json(value: Any) -> Iterator[str]: + if isinstance(value, dict): + blob = value.get("Bytes") + if isinstance(blob, str): + yield blob + for child in value.values(): + yield from walk_json(child) + elif isinstance(value, list): + for child in value: + yield from walk_json(child) + + +def finite_bounds(coords: list[tuple[float, float, float]]) -> tuple[float, float, float, float, float, float] | None: + finite = [coord for coord in coords if all(math.isfinite(part) for part in coord)] + if not finite: + return None + + xs = [coord[0] for coord in finite] + ys = [coord[1] for coord in finite] + zs = [coord[2] for coord in finite] + return min(xs), min(ys), min(zs), max(xs), max(ys), max(zs) + + +def parse_vand(path: Path, ordinal: int, data: bytes) -> VandBlob | None: + if len(data) < HEADER_SIZE or not data.startswith(b"VAND"): + return None + + version = struct.unpack_from(" len(data): + return VandBlob( + path=path, + ordinal=ordinal, + size=len(data), + version=version, + tile_x=tile_x, + tile_y=tile_y, + point_count=point_count, + coord_count=coord_count, + aux_count=aux_count, + point_count_2=point_count_2, + aux_count_2=aux_count_2, + point_count_3=point_count_3, + class_counts=collections.Counter({-1: point_count}), + raw_class_counts=collections.Counter(), + mask_counts=collections.Counter(), + bounds=None, + ) + + coords: list[tuple[float, float, float]] = [] + for index in range(coord_count): + coords.append(struct.unpack_from(" Iterator[VandBlob]: + document = json.loads(path.read_text(encoding="utf-8")) + ordinal = 0 + for encoded in walk_json(document): + try: + data = base64.b64decode(encoded) + except ValueError: + continue + blob = parse_vand(path, ordinal, data) + if blob: + yield blob + ordinal += 1 + + +def fmt_counter(counter: collections.Counter[int], limit: int) -> str: + return " ".join(f"{key}:{value}" for key, value in counter.most_common(limit)) + + +def fmt_bounds(bounds: tuple[float, float, float, float, float, float] | None) -> str: + if not bounds: + return "" + return ( + f"({bounds[0]:.1f},{bounds[1]:.1f},{bounds[2]:.1f}).." + f"({bounds[3]:.1f},{bounds[4]:.1f},{bounds[5]:.1f})" + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("json_files", nargs="+", type=Path) + parser.add_argument("--detail", type=int, default=12) + parser.add_argument("--counter-limit", type=int, default=12) + args = parser.parse_args() + + blobs: list[VandBlob] = [] + for path in args.json_files: + blobs.extend(iter_vand_blobs(path)) + + total_points = sum(blob.point_count for blob in blobs) + total_coords = sum(blob.coord_count for blob in blobs) + class_counts: collections.Counter[int] = collections.Counter() + raw_counts: collections.Counter[int] = collections.Counter() + mask_counts: collections.Counter[int] = collections.Counter() + versions: collections.Counter[int] = collections.Counter() + files: collections.Counter[str] = collections.Counter() + for blob in blobs: + class_counts.update(blob.class_counts) + raw_counts.update(blob.raw_class_counts) + mask_counts.update(blob.mask_counts) + versions[blob.version] += 1 + files[str(blob.path)] += 1 + + print(f"files={len(files)} blobs={len(blobs)} points={total_points} coords={total_coords}") + print(f"versions={dict(sorted(versions.items()))}") + print(f"classes={fmt_counter(class_counts, args.counter_limit)}") + print(f"raw13={fmt_counter(raw_counts, args.counter_limit)}") + print(f"masks={fmt_counter(mask_counts, args.counter_limit)}") + print("files:") + for path, count in files.most_common(): + print(f" {path}: {count}") + + if args.detail: + print("largest blobs:") + for blob in sorted(blobs, key=lambda item: item.point_count, reverse=True)[: args.detail]: + print( + f" {blob.path.name}#{blob.ordinal} size={blob.size} tile=({blob.tile_x},{blob.tile_y}) " + f"points={blob.point_count} coords={blob.coord_count} aux={blob.aux_count} " + f"classes={fmt_counter(blob.class_counts, args.counter_limit)} " + f"raw13={fmt_counter(blob.raw_class_counts, args.counter_limit)} " + f"bounds={fmt_bounds(blob.bounds)}" + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())