Probe native GPS query helpers

docs/traffic-system-debrief.md

@@ -4,6 +4,67 @@ This is based on the public NativeDB RTTI dump, WolvenKit exports from the local
 Steam install, and the generated `all.traffic_persistent` resource from the
 Phantom Liberty archive.
 
+## Current Working Theory
+
+The first hypothesis was that the in-game car GPS routes over the generated
+traffic lane graph and consumes lane metadata such as `maxSpeed`, highway flags,
+and lane connection probabilities. Live tests have not supported that:
+
+- boosting highway `maxSpeed` changed freeway traffic speed but did not change
+  player GPS route choice
+- changing `all.lane_connections` exit probabilities did not change player GPS
+  route choice
+- probing the obvious `TrafficSystem_Pathfinding` and
+  `StartPathfinding`/`StopPathfinding` string-xref wrappers did not fire during
+  deliberate world-map route plotting
+
+The stronger current theory is that player GPS route selection goes through a
+native GPS/mappin query path, with traffic data used by traffic simulation and
+possibly as an input to cooked GPS data, but not as a live route-cost graph that
+can be steered by simple resource weighting.
+
+## Static GPS Query Candidates
+
+Static disassembly of `Cyberpunk2077.exe` found script/native registration
+thunks for `RunGPSQuery` and `UpdateGPSQuery`. The string xrefs themselves are
+registration code, but their registered function pointers lead to more
+interesting native wrappers:
+
+```text
+RunGPSQuery string rva:       0x2cd9978
+RunGPSQuery registration xref: 0x147d068
+RunGPSQuery wrapper rva:      0x29bd5ac
+RunGPSQuery helper rva:       0x29bcf14
+
+UpdateGPSQuery string rva:       0x2cd9968
+UpdateGPSQuery registration xref: 0x147cfb4
+UpdateGPSQuery wrapper rva:      0x29bd6c8
+UpdateGPSQuery helper rva:       0x29bd254
+```
+
+`RunGPSQueryHelper` appears to read source/target vector-ish script arguments,
+resolve a target object/mappin, build temporary bitsets/sets, and call helpers
+near `0x204abdc` and `0x204ac1c`. Its return value is used as a success flag by
+the wrapper.
+
+`UpdateGPSQueryHelper` appears related but takes a request type and target. Its
+return value is also used as a success flag.
+
+The next RED4ext probe should hook these helpers with return-value-preserving
+signatures, not the registration thunks:
+
+```cpp
+bool RunGPSQueryHelper(void* this_, void* from, void* to, void* debugText,
+                       void* resultText, float maxDistance);
+bool UpdateGPSQueryHelper(void* this_, uint32_t requestType, void* target,
+                          void* debugText);
+```
+
+These signatures are inferred from the wrapper call sites and Windows x64
+calling convention. If either helper fires exactly when a map destination is
+selected, it is a much better path into the actual route computation than the
+traffic-pathfinding probes.
+
 ## The High-Level Shape
 
 Cyberpunk appears to split "navigation" into at least two major domains:

red4ext/EdgeWeightGPS/src/Main.cpp

@@ -92,10 +92,8 @@ namespace
 constexpr uint32_t kGameStateRunning = 2;
 constexpr uintptr_t kGpsSystemTickRva = 0x0E6B62C;
 constexpr uintptr_t kTrafficSystemPathfindingRva = 0x0AD1234;
-constexpr uintptr_t kStopPathfindingA = 0x11C20A8;
+constexpr uintptr_t kRunGpsQueryHelperRva = 0x29BCF14;
-constexpr uintptr_t kStopPathfindingB = 0x11C2148;
+constexpr uintptr_t kUpdateGpsQueryHelperRva = 0x29BD254;
-constexpr uintptr_t kStartPathfindingA = 0x11C21FC;
-constexpr uintptr_t kStartPathfindingB = 0x11C22B0;
 
 HMODULE gModule = nullptr;
 std::mutex gLogMutex;
@@ -105,20 +103,18 @@ uint32_t gUpdateLogCount = 0;
 bool gScannedExecutable = false;
 uint32_t gGpsTickLogCount = 0;
 uint32_t gTrafficPathfindingLogCount = 0;
-uint32_t gStopPathfindingALogCount = 0;
+uint32_t gRunGpsQueryLogCount = 0;
-uint32_t gStopPathfindingBLogCount = 0;
+uint32_t gUpdateGpsQueryLogCount = 0;
-uint32_t gStartPathfindingALogCount = 0;
-uint32_t gStartPathfindingBLogCount = 0;
 uint64_t gLogStartTick = 0;
 uint64_t gLogFrequency = 0;
 
 using ProbeFunc = void (*)(void* aThis, void* a2, void* a3, void* a4);
+using RunGpsQueryFunc = bool (*)(void* aThis, void* aFrom, void* aTo, void* aDebugText, void* aResultText, float aMaxDistance);
+using UpdateGpsQueryFunc = bool (*)(void* aThis, uint32_t aRequestType, void* aTarget, void* aDebugText);
 ProbeFunc gOriginalGpsSystemTick = nullptr;
 ProbeFunc gOriginalTrafficSystemPathfinding = nullptr;
-ProbeFunc gOriginalStopPathfindingA = nullptr;
+RunGpsQueryFunc gOriginalRunGpsQueryHelper = nullptr;
-ProbeFunc gOriginalStopPathfindingB = nullptr;
+UpdateGpsQueryFunc gOriginalUpdateGpsQueryHelper = nullptr;
-ProbeFunc gOriginalStartPathfindingA = nullptr;
-ProbeFunc gOriginalStartPathfindingB = nullptr;
 
 std::filesystem::path GetLogPath()
 {
@@ -275,10 +271,13 @@ void ScanExecutableStrings()
     header << "exe-string scan begin base=0x" << std::hex << imageBase << " size=0x" << imageSize << std::dec;
     LogRed4ext(header.str());
 
-    constexpr std::array<const char*, 18> kNeedles{
+    constexpr std::array<const char*, 21> kNeedles{
         "GPSSystem/Tick",
         "GPSSystem",
         "GPSSettings",
+        "RunGPSQuery",
+        "UpdateGPSQuery",
+        "gameuiGPSGameController",
         "OnSetHasGPSPortal",
         "IsGPSPortal",
         "m_cookedGpsData",
@@ -289,8 +288,6 @@ void ScanExecutableStrings()
         "Pathfinding Algorithm Failed",
         "No Path Found in Traffic",
         "PathFindingFailed",
-        "StartPathfinding",
-        "StopPathfinding",
         "worldTrafficLanePlayerGPSInfo",
         "worldTrafficPersistentResource",
         "worldTrafficConnectivityOutLane",
@@ -369,60 +366,47 @@ void DetourTrafficSystemPathfinding(void* aThis, void* a2, void* a3, void* a4)
     }
 }
 
-void DetourStopPathfindingA(void* aThis, void* a2, void* a3, void* a4)
+bool DetourRunGpsQueryHelper(void* aThis, void* aFrom, void* aTo, void* aDebugText, void* aResultText,
+                             float aMaxDistance)
 {
-    if (gStopPathfindingALogCount < 32)
+    bool result = false;
+    if (gOriginalRunGpsQueryHelper)
     {
-        LogHookCall("StopPathfindingA", gStopPathfindingALogCount, aThis, a2, a3, a4);
+        result = gOriginalRunGpsQueryHelper(aThis, aFrom, aTo, aDebugText, aResultText, aMaxDistance);
-        ++gStopPathfindingALogCount;
     }
 
-    if (gOriginalStopPathfindingA)
+    if (gRunGpsQueryLogCount < 64)
     {
-        gOriginalStopPathfindingA(aThis, a2, a3, a4);
+        std::ostringstream line;
+        line << "hook RunGPSQueryHelper call=" << gRunGpsQueryLogCount << " this=" << aThis << " from=" << aFrom
+             << " to=" << aTo << " debugText=" << aDebugText << " resultText=" << aResultText
+             << " maxDistance=" << aMaxDistance << " result=" << (result ? "true" : "false");
+        LogRed4ext(line.str());
+        ++gRunGpsQueryLogCount;
     }
+
+    return result;
 }
 
-void DetourStopPathfindingB(void* aThis, void* a2, void* a3, void* a4)
+bool DetourUpdateGpsQueryHelper(void* aThis, uint32_t aRequestType, void* aTarget, void* aDebugText)
 {
-    if (gStopPathfindingBLogCount < 32)
+    bool result = false;
+    if (gOriginalUpdateGpsQueryHelper)
     {
-        LogHookCall("StopPathfindingB", gStopPathfindingBLogCount, aThis, a2, a3, a4);
+        result = gOriginalUpdateGpsQueryHelper(aThis, aRequestType, aTarget, aDebugText);
-        ++gStopPathfindingBLogCount;
     }
 
-    if (gOriginalStopPathfindingB)
+    if (gUpdateGpsQueryLogCount < 64)
     {
-        gOriginalStopPathfindingB(aThis, a2, a3, a4);
+        std::ostringstream line;
-    }
+        line << "hook UpdateGPSQueryHelper call=" << gUpdateGpsQueryLogCount << " this=" << aThis
-}
+             << " requestType=" << aRequestType << " target=" << aTarget << " debugText=" << aDebugText
-
+             << " result=" << (result ? "true" : "false");
-void DetourStartPathfindingA(void* aThis, void* a2, void* a3, void* a4)
+        LogRed4ext(line.str());
-{
+        ++gUpdateGpsQueryLogCount;
-    if (gStartPathfindingALogCount < 32)
-    {
-        LogHookCall("StartPathfindingA", gStartPathfindingALogCount, aThis, a2, a3, a4);
-        ++gStartPathfindingALogCount;
     }
 
-    if (gOriginalStartPathfindingA)
+    return result;
-    {
-        gOriginalStartPathfindingA(aThis, a2, a3, a4);
-    }
-}
-
-void DetourStartPathfindingB(void* aThis, void* a2, void* a3, void* a4)
-{
-    if (gStartPathfindingBLogCount < 32)
-    {
-        LogHookCall("StartPathfindingB", gStartPathfindingBLogCount, aThis, a2, a3, a4);
-        ++gStartPathfindingBLogCount;
-    }
-
-    if (gOriginalStartPathfindingB)
-    {
-        gOriginalStartPathfindingB(aThis, a2, a3, a4);
-    }
 }
 
 void AttachProbeHook(const char* aName, uintptr_t aRva, void* aDetour, void** aOriginal)
@@ -534,24 +518,19 @@ RED4EXT_C_EXPORT bool RED4EXT_CALL Main(RED4ext::v1::PluginHandle aHandle, RED4e
         AttachProbeHook("TrafficSystem_Pathfinding", kTrafficSystemPathfindingRva,
                         reinterpret_cast<void*>(&DetourTrafficSystemPathfinding),
                         reinterpret_cast<void**>(&gOriginalTrafficSystemPathfinding));
-        AttachProbeHook("StopPathfindingA", kStopPathfindingA, reinterpret_cast<void*>(&DetourStopPathfindingA),
+        AttachProbeHook("RunGPSQueryHelper", kRunGpsQueryHelperRva, reinterpret_cast<void*>(&DetourRunGpsQueryHelper),
-                        reinterpret_cast<void**>(&gOriginalStopPathfindingA));
+                        reinterpret_cast<void**>(&gOriginalRunGpsQueryHelper));
-        AttachProbeHook("StopPathfindingB", kStopPathfindingB, reinterpret_cast<void*>(&DetourStopPathfindingB),
+        AttachProbeHook("UpdateGPSQueryHelper", kUpdateGpsQueryHelperRva,
-                        reinterpret_cast<void**>(&gOriginalStopPathfindingB));
+                        reinterpret_cast<void*>(&DetourUpdateGpsQueryHelper),
-        AttachProbeHook("StartPathfindingA", kStartPathfindingA, reinterpret_cast<void*>(&DetourStartPathfindingA),
+                        reinterpret_cast<void**>(&gOriginalUpdateGpsQueryHelper));
-                        reinterpret_cast<void**>(&gOriginalStartPathfindingA));
-        AttachProbeHook("StartPathfindingB", kStartPathfindingB, reinterpret_cast<void*>(&DetourStartPathfindingB),
-                        reinterpret_cast<void**>(&gOriginalStartPathfindingB));
     }
 
     if (aReason == RED4ext::v1::EMainReason::Unload)
     {
         DetachProbeHook("GPSSystem/Tick", kGpsSystemTickRva);
         DetachProbeHook("TrafficSystem_Pathfinding", kTrafficSystemPathfindingRva);
-        DetachProbeHook("StopPathfindingA", kStopPathfindingA);
+        DetachProbeHook("RunGPSQueryHelper", kRunGpsQueryHelperRva);
-        DetachProbeHook("StopPathfindingB", kStopPathfindingB);
+        DetachProbeHook("UpdateGPSQueryHelper", kUpdateGpsQueryHelperRva);
-        DetachProbeHook("StartPathfindingA", kStartPathfindingA);
-        DetachProbeHook("StartPathfindingB", kStartPathfindingB);
     }
 
     return true;

tools/find_route_static_candidates.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+"""Find static GPS/map route candidates in a Cyberpunk 2077 executable."""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import json
+import re
+import struct
+from pathlib import Path
+
+
+@dataclasses.dataclass(frozen=True)
+class Section:
+    name: str
+    virtual_address: int
+    virtual_size: int
+    raw_offset: int
+    raw_size: int
+    characteristics: int
+
+    @property
+    def is_code(self) -> bool:
+        return bool(self.characteristics & 0x20) and bool(self.characteristics & 0x20000000)
+
+    @property
+    def is_readable_data(self) -> bool:
+        return bool(self.characteristics & 0x40000000) and not self.is_code
+
+    def contains_rva(self, rva: int) -> bool:
+        return self.virtual_address <= rva < self.virtual_address + max(self.virtual_size, self.raw_size)
+
+    def rva_to_offset(self, rva: int) -> int:
+        return self.raw_offset + (rva - self.virtual_address)
+
+
+def parse_pe(data: bytes) -> tuple[int, list[Section]]:
+    if data[:2] != b"MZ":
+        raise ValueError("not an MZ executable")
+
+    pe_offset = struct.unpack_from("<I", data, 0x3C)[0]
+    if data[pe_offset : pe_offset + 4] != b"PE\0\0":
+        raise ValueError("not a PE executable")
+
+    coff_offset = pe_offset + 4
+    section_count = struct.unpack_from("<H", data, coff_offset + 2)[0]
+    optional_size = struct.unpack_from("<H", data, coff_offset + 16)[0]
+    optional_offset = coff_offset + 20
+    if struct.unpack_from("<H", data, optional_offset)[0] != 0x20B:
+        raise ValueError("expected PE32+ executable")
+
+    image_base = struct.unpack_from("<Q", data, optional_offset + 24)[0]
+    section_offset = optional_offset + optional_size
+    sections: list[Section] = []
+    for index in range(section_count):
+        offset = section_offset + index * 40
+        name = data[offset : offset + 8].split(b"\0", 1)[0].decode("ascii", "replace")
+        virtual_size, virtual_address, raw_size, raw_offset = struct.unpack_from("<IIII", data, offset + 8)
+        characteristics = struct.unpack_from("<I", data, offset + 36)[0]
+        sections.append(Section(name, virtual_address, virtual_size, raw_offset, raw_size, characteristics))
+
+    return image_base, sections
+
+
+def iter_strings(data: bytes, section: Section, min_len: int) -> list[tuple[int, str, str]]:
+    results: list[tuple[int, str, str]] = []
+    start = section.raw_offset
+    end = min(len(data), section.raw_offset + section.raw_size)
+    blob = data[start:end]
+
+    ascii_re = re.compile(rb"[\x20-\x7e]{%d,}" % min_len)
+    for match in ascii_re.finditer(blob):
+        text = match.group(0).decode("ascii", "replace")
+        results.append((section.virtual_address + match.start(), "ascii", text))
+
+    utf16_re = re.compile((rb"(?:[\x20-\x7e]\x00){%d,}" % min_len))
+    for match in utf16_re.finditer(blob):
+        raw = match.group(0)
+        text = raw.decode("utf-16le", "replace")
+        results.append((section.virtual_address + match.start(), "utf16", text))
+
+    return results
+
+
+def find_xrefs(data: bytes, sections: list[Section], image_base: int, target_rvas: set[int]) -> dict[int, list[int]]:
+    target_vas = {image_base + rva: rva for rva in target_rvas}
+    hits = {rva: [] for rva in target_rvas}
+    for section in sections:
+        if not section.is_code:
+            continue
+
+        start = section.raw_offset
+        end = min(len(data), section.raw_offset + section.raw_size)
+        for offset in range(start, max(start, end - 4)):
+            disp = struct.unpack_from("<i", data, offset)[0]
+            instr_end_va = image_base + section.virtual_address + (offset - section.raw_offset) + 4
+            target_va = instr_end_va + disp
+            target_rva = target_vas.get(target_va)
+            if target_rva is not None:
+                hits[target_rva].append(section.virtual_address + (offset - section.raw_offset))
+    return hits
+
+
+def load_address_symbols(path: Path, pattern: re.Pattern[str]) -> list[tuple[int, str]]:
+    payload = json.loads(path.read_text())
+    symbols: list[tuple[int, str]] = []
+    for item in payload.get("Addresses", []):
+        symbol = item.get("symbol")
+        offset = item.get("offset")
+        if not symbol or not offset or not pattern.search(symbol):
+            continue
+
+        section, _, value = offset.partition(":")
+        if section != "0001":
+            continue
+
+        symbols.append((int(value, 16), symbol))
+    return sorted(symbols)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("exe", type=Path)
+    parser.add_argument("--addresses", type=Path)
+    parser.add_argument(
+        "--pattern",
+        default=r"gps|route|path|mappin|track|quest|journal|navigation|map",
+        help="case-insensitive regex for string and symbol filtering",
+    )
+    parser.add_argument("--min-len", type=int, default=5)
+    parser.add_argument("--max-strings", type=int, default=300)
+    args = parser.parse_args()
+
+    data = args.exe.read_bytes()
+    image_base, sections = parse_pe(data)
+    pattern = re.compile(args.pattern, re.IGNORECASE)
+
+    matches: list[tuple[int, str, str]] = []
+    for section in sections:
+        if section.is_readable_data:
+            matches.extend(item for item in iter_strings(data, section, args.min_len) if pattern.search(item[2]))
+
+    matches.sort(key=lambda item: (item[0], item[1], item[2]))
+    target_rvas = {rva for rva, _, _ in matches}
+    xrefs = find_xrefs(data, sections, image_base, target_rvas)
+
+    print(f"image_base=0x{image_base:x}")
+    print(f"string_matches={len(matches)} pattern={args.pattern!r}")
+    for rva, encoding, text in matches[: args.max_strings]:
+        hit_text = ", ".join(f"0x{hit:x}" for hit in xrefs.get(rva, [])[:12])
+        if len(xrefs.get(rva, [])) > 12:
+            hit_text += f", ... +{len(xrefs[rva]) - 12}"
+        print(f"string rva=0x{rva:x} enc={encoding} xrefs={len(xrefs.get(rva, []))} text={text!r}")
+        if hit_text:
+            print(f"  xref_disp_rvas={hit_text}")
+
+    if args.addresses:
+        print()
+        symbols = load_address_symbols(args.addresses, pattern)
+        print(f"address_symbols={len(symbols)}")
+        for rva, symbol in symbols[: args.max_strings]:
+            print(f"symbol rva=0x{rva:x} name={symbol}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())