Fix traffic archive extraction workflow

docs/traffic-system-debrief.md

@@ -129,36 +129,37 @@ The important flags for GPS weighting are `Road`, `Intersection`, `GPSOnly`, and
 
 ## Observed Lane Distribution
 
-The local EP1 `all.traffic_persistent` export contains `31,717` lanes.
+The local EP1 `all.traffic_persistent` resource contains `33,952` lanes when
+extracted from the archive's raw compressed segment.
 
 Observed `maxSpeed` distribution:
 
 ```text
 all lanes:
-  1: 12284
+  1: 12852
-  15: 11638
+  15: 12107
-  10: 4675
+  10: 5859
-  17: 3013
+  17: 3027
   14: 96
   8: 8
   9: 2
   4: 1
 
 highway lanes:
-  17: 2688
+  17: 2700
   15: 164
   1: 2
 
 road lanes:
-  15: 11474
+  15: 11943
-  10: 495
+  10: 671
-  17: 325
+  17: 327
   14: 96
   8: 4
 
 pavement lanes:
-  1: 12280
+  1: 12848
-  10: 4180
+  10: 5188
   8: 4
   9: 2
   4: 1
@@ -172,10 +173,10 @@ defaults:
 - highway speed target: `25`
 - road speed floor: `15`
 
-That changes `2,506` lanes:
+That changes `2,519` lanes:
 
-- `2,419` highway boosts
+- `2,429` highway boosts
-- `87` road floor boosts
+- `90` road floor boosts
 
 It leaves pavement/crosswalk/blocked/disabled lanes alone.
 
@@ -226,13 +227,21 @@ already consumes.
 
 The working flow is:
 
-1. Extract `all.traffic_persistent` with WolvenKit CLI.
+1. Read the archive index and extract the exact compressed segment for resource
-2. Serialize the CR2W resource to JSON.
+   hash `3419764573789342681`.
-3. Patch lane `maxSpeed` values.
+2. Decompress that KARK segment with WolvenKit CLI's Oodle command.
-4. Deserialize the patched JSON back to CR2W.
+3. Serialize the CR2W resource to JSON.
-5. Put the CR2W back under the original resource path.
+4. Patch lane `maxSpeed` values.
-6. Pack an archive.
+5. Deserialize the patched JSON back to CR2W.
-7. Place the archive in `archive/pc/mod`.
+6. Put the CR2W back under the original resource path.
+7. Pack an archive.
+8. Place the archive in `archive/pc/mod`.
+
+Do not use plain `cp77tools extract` for this resource. In this install,
+WolvenKit CLI extracted a `9,984,106` byte CR2W, while the stock archive segment
+declares and decompresses to `10,673,648` bytes. A vanilla control archive built
+from the smaller extraction crashed during load. A vanilla control archive built
+from the raw segment reached the game and the GPS worked normally.
 
 The generated archive currently contains exactly:
 
@@ -249,8 +258,9 @@ and traffic simulation, the patch may also affect some traffic behavior on
 highways.
 
 The Oodle library was not available inside the toolbox, so WolvenKit packed with
-its Kraken fallback. WolvenKit successfully round-tripped and listed the archive,
+its Kraken fallback. A vanilla full-resource control archive loaded in game and
-but in-game testing is still required.
+GPS worked normally, so the earlier crash was caused by bad extracted data rather
+than the Kraken-packed archive container.
 
 The patch targets the EP1 traffic resource because Phantom Liberty is installed.
 On a non-PL install, the basegame archive resource would need to be patched

tools/build_lane_weight_archive.sh

@@ -21,13 +21,24 @@ pack_root="${work_dir}/${mod_name}"
 packed_dir="${work_dir}/05_packed"
 resource_dir=$(dirname "${resource_path}")
 resource_file=$(basename "${resource_path}")
+resource_hash=3419764573789342681
 
-mkdir -p "${raw_dir}" "${json_dir}" "${patched_json_dir}" "${patched_raw_dir}" "${pack_root}/${resource_dir}" "${packed_dir}"
+mkdir -p "${raw_dir}/${resource_dir}" "${json_dir}" "${patched_json_dir}" "${patched_raw_dir}" "${pack_root}/${resource_dir}" "${packed_dir}"
 
-tools/cp77_toolbox.sh extract "${archive_path}" \
+if [[ -f "${archive_path}" ]]; then
-  --gamepath "${game_dir}" \
+  kark_path="${raw_dir}/${resource_file}.kark"
-  --outpath "${raw_dir}" \
+  decompressed_base="${raw_dir}/${resource_file}.raw"
-  --regex "${resource_regex}"
+  python3 tools/extract_archive_segment.py "${archive_path}" "${resource_hash}" "${kark_path}"
+  tools/cp77_toolbox.sh oodle decompress "${kark_path}" "${decompressed_base}" || true
+  if [[ ! -f "${decompressed_base}.bin" ]]; then
+    echo "failed to decompress ${kark_path}" >&2
+    exit 1
+  fi
+  mv "${decompressed_base}.bin" "${raw_dir}/${resource_path}"
+else
+  echo "raw segment extraction needs a single archive file, got: ${archive_path}" >&2
+  exit 2
+fi
 
 tools/cp77_toolbox.sh convert serialize "${raw_dir}" --outpath "${json_dir}"
 

tools/extract_archive_segment.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Extract a single compressed RED4 archive segment by resource hash."""
+
+from __future__ import annotations
+
+import argparse
+import struct
+from pathlib import Path
+
+
+FILE_ENTRY_SIZE = 56
+FILE_SEGMENT_SIZE = 16
+
+
+def parse_hash(value: str) -> int:
+    return int(value, 0)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("archive", type=Path)
+    parser.add_argument("resource_hash", type=parse_hash)
+    parser.add_argument("outpath", type=Path)
+    args = parser.parse_args()
+
+    with args.archive.open("rb") as archive:
+        header = archive.read(40)
+        if len(header) != 40:
+            raise SystemExit("archive header is too short")
+        magic, _version, index_pos, _index_size, *_rest = struct.unpack("<IIQIQIQ", header)
+        if magic != 1380009042:
+            raise SystemExit(f"unexpected archive magic: {magic}")
+
+        archive.seek(index_pos + 16)
+        file_count, segment_count, _dependency_count = struct.unpack("<III", archive.read(12))
+
+        found: tuple[int, int] | None = None
+        for _ in range(file_count):
+            entry = archive.read(FILE_ENTRY_SIZE)
+            (
+                key,
+                _timestamp,
+                _flags,
+                segments_start,
+                segments_end,
+                _dependencies_start,
+                _dependencies_end,
+                _sha1,
+            ) = struct.unpack("<QqIIIII20s", entry)
+            if key == args.resource_hash:
+                if segments_end - segments_start != 1:
+                    raise SystemExit(
+                        f"resource has {segments_end - segments_start} segments; expected 1"
+                    )
+                found = (segments_start, segments_end)
+
+        if found is None:
+            raise SystemExit(f"resource hash {args.resource_hash} not found")
+
+        segments = []
+        for _ in range(segment_count):
+            segments.append(struct.unpack("<QII", archive.read(FILE_SEGMENT_SIZE)))
+
+        offset, z_size, size = segments[found[0]]
+        archive.seek(offset)
+        data = archive.read(z_size)
+        if len(data) != z_size:
+            raise SystemExit("archive ended before segment data was fully read")
+
+    args.outpath.parent.mkdir(parents=True, exist_ok=True)
+    args.outpath.write_bytes(data)
+    print(f"wrote {args.outpath} z_size={z_size} size={size}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())