376 lines
13 KiB
Python
Executable File
376 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Summarize EdgeWeightGPS route-result records from RED4ext logs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import collections
|
|
import dataclasses
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
RE_TIMESTAMP = re.compile(r"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{3})")
|
|
RE_ELAPSED_MS = re.compile(r" \+(\d+)ms ")
|
|
RE_RECORD_BLOCK = re.compile(r"gpsResult_ptr28_rec40=\[(.*?)\]")
|
|
RE_RECORD = re.compile(r"(\d+):([0-9a-f]+),0x([0-9a-f]{8}),([^;\]]+)")
|
|
RE_JOB_COUNT = re.compile(r"GPSRouteJobBuild result call=(\d+).*?job_count50=(\d+)/0x([0-9a-f]+)")
|
|
RE_FIELD_DEC_HEX = re.compile(r"([A-Za-z0-9_]+)=(-?\d+)/0x([0-9a-fA-F]+)")
|
|
RE_FIELD_HEX = re.compile(r"([A-Za-z0-9_]+)=0x([0-9a-fA-F]+)")
|
|
RE_FIELD_DEC = re.compile(r"([A-Za-z0-9_]+)=(-?\d+)(?= |$)")
|
|
|
|
FLAGS = {
|
|
"FromRoadSpline": 1,
|
|
"Bidirectional": 2,
|
|
"PatrolRoute": 4,
|
|
"Pavement": 8,
|
|
"Road": 16,
|
|
"Intersection": 32,
|
|
"NeverDeadEnd": 64,
|
|
"TrafficDisabled": 128,
|
|
"CrossWalk": 256,
|
|
"GPSOnly": 512,
|
|
"ShowDebug": 1024,
|
|
"Blockade": 2048,
|
|
"Yield": 4096,
|
|
"NoAIDriving": 8192,
|
|
"Highway": 16384,
|
|
"NoAutodrive": 32768,
|
|
}
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class SubmitInfo:
|
|
line_no: int
|
|
timestamp: str
|
|
elapsed_ms: int | None
|
|
call: int
|
|
fields: dict[str, int]
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class RouteRecord:
|
|
index: int
|
|
handle: int
|
|
packed: int
|
|
parts: list[str]
|
|
|
|
@property
|
|
def low_class(self) -> int:
|
|
return self.packed & 0xFF
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class RouteResult:
|
|
line_no: int
|
|
timestamp: str
|
|
elapsed_ms: int | None
|
|
query_id: int | None
|
|
submit_call: int | None
|
|
submit_ret_rva: int | None
|
|
fields: dict[str, int]
|
|
records: list[RouteRecord]
|
|
|
|
|
|
def bytes_le(value: int) -> tuple[int, int, int, int]:
|
|
return tuple((value >> (8 * index)) & 0xFF for index in range(4))
|
|
|
|
|
|
def flag_value(flags: Any) -> int:
|
|
if isinstance(flags, dict):
|
|
return int(flags.get("Value", flags.get("$value", 0)) or 0)
|
|
return int(flags or 0)
|
|
|
|
|
|
def lane_category(flags: int) -> str:
|
|
if flags & FLAGS["Highway"]:
|
|
return "highway"
|
|
if flags & FLAGS["GPSOnly"]:
|
|
return "gpsonly"
|
|
if flags & FLAGS["Road"]:
|
|
return "road"
|
|
if flags & FLAGS["Pavement"]:
|
|
return "pavement"
|
|
return "other"
|
|
|
|
|
|
def parse_timestamp(line: str) -> str:
|
|
match = RE_TIMESTAMP.search(line)
|
|
return match.group(1) if match else "<unknown>"
|
|
|
|
|
|
def parse_elapsed_ms(line: str) -> int | None:
|
|
match = RE_ELAPSED_MS.search(line)
|
|
return int(match.group(1)) if match else None
|
|
|
|
|
|
def parse_fields(line: str) -> dict[str, int]:
|
|
fields: dict[str, int] = {}
|
|
for match in RE_FIELD_DEC_HEX.finditer(line):
|
|
fields[match.group(1)] = int(match.group(2))
|
|
for match in RE_FIELD_HEX.finditer(line):
|
|
fields.setdefault(match.group(1), int(match.group(2), 16))
|
|
for match in RE_FIELD_DEC.finditer(line):
|
|
fields.setdefault(match.group(1), int(match.group(2)))
|
|
return fields
|
|
|
|
|
|
def load_lane_lookup(path: Path | None) -> dict[int, dict[str, Any]]:
|
|
if not path:
|
|
return {}
|
|
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
lanes = data["Data"]["RootChunk"]["data"]["lanes"]
|
|
lookup: dict[int, dict[str, Any]] = {}
|
|
for index, lane in enumerate(lanes):
|
|
node_hash = lane.get("nodeRefHash")
|
|
if node_hash is None:
|
|
continue
|
|
flags = flag_value(lane.get("flags"))
|
|
lookup[int(node_hash)] = {
|
|
"index": index,
|
|
"flags": flags,
|
|
"category": lane_category(flags),
|
|
"speed": flag_value(lane.get("maxSpeed")),
|
|
"length": lane.get("length"),
|
|
}
|
|
return lookup
|
|
|
|
|
|
def parse_route_records(block: str) -> list[RouteRecord]:
|
|
records: list[RouteRecord] = []
|
|
for record_match in RE_RECORD.finditer(block):
|
|
pieces = record_match.group(4).split(",")
|
|
if len(pieces) < 4:
|
|
continue
|
|
records.append(
|
|
RouteRecord(
|
|
index=int(record_match.group(1)),
|
|
handle=int(record_match.group(2), 16),
|
|
packed=int(record_match.group(3), 16),
|
|
parts=pieces,
|
|
)
|
|
)
|
|
return records
|
|
|
|
|
|
def iter_route_results(path: Path) -> tuple[dict[int, SubmitInfo], list[RouteResult], list[int]]:
|
|
submits: dict[int, SubmitInfo] = {}
|
|
results: list[RouteResult] = []
|
|
job_counts: list[int] = []
|
|
|
|
for line_no, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), start=1):
|
|
fields = parse_fields(line)
|
|
if "hook GPSQuerySubmit 0x70a42c" in line:
|
|
call = fields.get("call")
|
|
if call is not None:
|
|
submits[call] = SubmitInfo(
|
|
line_no=line_no,
|
|
timestamp=parse_timestamp(line),
|
|
elapsed_ms=parse_elapsed_ms(line),
|
|
call=call,
|
|
fields=fields,
|
|
)
|
|
|
|
job_match = RE_JOB_COUNT.search(line)
|
|
if job_match:
|
|
job_counts.append(int(job_match.group(2)))
|
|
|
|
block_match = RE_RECORD_BLOCK.search(line)
|
|
if not block_match:
|
|
continue
|
|
|
|
records = parse_route_records(block_match.group(1))
|
|
if not records:
|
|
continue
|
|
|
|
results.append(
|
|
RouteResult(
|
|
line_no=line_no,
|
|
timestamp=parse_timestamp(line),
|
|
elapsed_ms=parse_elapsed_ms(line),
|
|
query_id=fields.get("queryId"),
|
|
submit_call=fields.get("submitCall"),
|
|
submit_ret_rva=fields.get("submitRetRva"),
|
|
fields=fields,
|
|
records=records,
|
|
)
|
|
)
|
|
|
|
return submits, results, job_counts
|
|
|
|
|
|
def summarize_route_result(route: RouteResult, lane_lookup: dict[int, dict[str, Any]]) -> tuple[dict[str, Any], int]:
|
|
class_counts = collections.Counter(record.low_class for record in route.records)
|
|
category_counts: collections.Counter[str] = collections.Counter()
|
|
category_lengths: collections.Counter[str] = collections.Counter()
|
|
speed_lengths: collections.Counter[int] = collections.Counter()
|
|
unmatched = 0
|
|
|
|
for record in route.records:
|
|
lane = lane_lookup.get(record.handle)
|
|
if not lane:
|
|
unmatched += 1
|
|
continue
|
|
category = lane["category"]
|
|
length = float(lane.get("length") or 0.0)
|
|
category_counts[category] += 1
|
|
category_lengths[category] += length
|
|
speed_lengths[int(lane["speed"])] += length
|
|
|
|
total_length = sum(category_lengths.values())
|
|
summary = {
|
|
"class_counts": class_counts,
|
|
"category_counts": category_counts,
|
|
"category_lengths": category_lengths,
|
|
"speed_lengths": speed_lengths,
|
|
"total_length": total_length,
|
|
"unmatched": unmatched,
|
|
}
|
|
return summary, len(route.records) - unmatched
|
|
|
|
|
|
def format_counter(counter: collections.Counter[Any], limit: int | None = None, digits: int | None = None) -> str:
|
|
items = counter.most_common(limit)
|
|
parts = []
|
|
for key, value in items:
|
|
if digits is None:
|
|
parts.append(f"{key}:{value}")
|
|
else:
|
|
parts.append(f"{key}:{value:.{digits}f}")
|
|
return "{" + ", ".join(parts) + "}"
|
|
|
|
|
|
def print_per_route(
|
|
path: Path,
|
|
submits: dict[int, SubmitInfo],
|
|
route_results: list[RouteResult],
|
|
lane_lookup: dict[int, dict[str, Any]],
|
|
) -> None:
|
|
print(" per-route results:")
|
|
for ordinal, route in enumerate(route_results, start=1):
|
|
submit = submits.get(route.submit_call) if route.submit_call is not None else None
|
|
summary, matched = summarize_route_result(route, lane_lookup)
|
|
span = route.fields.get("gpsResult_u20")
|
|
submit_fields = submit.fields if submit else {}
|
|
ret_rva = route.submit_ret_rva
|
|
ret_text = f"0x{ret_rva:x}" if ret_rva is not None else "?"
|
|
submit_ms = submit.elapsed_ms if submit else None
|
|
result_ms = route.elapsed_ms
|
|
duration_ms = result_ms - submit_ms if submit_ms is not None and result_ms is not None else None
|
|
submit_marker = f"+{submit_ms}ms" if submit_ms is not None else "?"
|
|
result_marker = f"+{result_ms}ms" if result_ms is not None else "?"
|
|
duration_marker = f"{duration_ms}ms" if duration_ms is not None else "?"
|
|
print(
|
|
f" route#{ordinal} qid={route.query_id} submitCall={route.submit_call} "
|
|
f"submit={submit_marker} result={result_marker} duration={duration_marker} line={route.line_no} "
|
|
f"ret={ret_text} records={len(route.records)} spanU20={span} matched={matched}"
|
|
)
|
|
if submit_fields:
|
|
print(
|
|
" query "
|
|
f"f08=0x{submit_fields.get('query_f08', 0):x} "
|
|
f"f0c={submit_fields.get('query_f0c')} "
|
|
f"fcc={submit_fields.get('query_fcc')}"
|
|
)
|
|
if lane_lookup:
|
|
category_lengths = summary["category_lengths"]
|
|
total = summary["total_length"]
|
|
print(
|
|
" resource "
|
|
f"segments={format_counter(summary['category_counts'])} "
|
|
f"length={format_counter(category_lengths, digits=1)} "
|
|
f"total={total:.1f} "
|
|
f"speedLength={format_counter(summary['speed_lengths'], limit=6, digits=1)} "
|
|
f"unmatched={summary['unmatched']}"
|
|
)
|
|
print(f" routeClasses={format_counter(summary['class_counts'])}")
|
|
|
|
|
|
def summarize(path: Path, lane_lookup: dict[int, dict[str, Any]]) -> None:
|
|
submits, route_results, job_counts = iter_route_results(path)
|
|
routes = 0
|
|
records = 0
|
|
first_byte = collections.Counter()
|
|
byte_positions: list[collections.Counter[int]] = [collections.Counter() for _ in range(4)]
|
|
patterns = collections.Counter()
|
|
segments = collections.Counter()
|
|
class_categories: dict[int, collections.Counter[str]] = collections.defaultdict(collections.Counter)
|
|
class_flags: dict[int, collections.Counter[int]] = collections.defaultdict(collections.Counter)
|
|
class_speed: dict[int, collections.Counter[int]] = collections.defaultdict(collections.Counter)
|
|
matched_handles = 0
|
|
unmatched_handles = 0
|
|
|
|
for route in route_results:
|
|
route_records = 0
|
|
for record in route.records:
|
|
route_records += 1
|
|
records += 1
|
|
bytes_tuple = bytes_le(record.packed)
|
|
class_id = bytes_tuple[0]
|
|
patterns[bytes_tuple] += 1
|
|
first_byte[class_id] += 1
|
|
for index, byte in enumerate(bytes_tuple):
|
|
byte_positions[index][byte] += 1
|
|
segments[(record.handle, class_id)] += 1
|
|
|
|
if lane_lookup:
|
|
lane = lane_lookup.get(record.handle)
|
|
if lane:
|
|
matched_handles += 1
|
|
class_categories[class_id][lane["category"]] += 1
|
|
class_flags[class_id][lane["flags"]] += 1
|
|
class_speed[class_id][lane["speed"]] += 1
|
|
else:
|
|
unmatched_handles += 1
|
|
|
|
if route_records:
|
|
routes += 1
|
|
|
|
print(f"{path}:")
|
|
print(f" routes={routes} records={records}")
|
|
if job_counts:
|
|
sorted_counts = sorted(job_counts)
|
|
p50 = sorted_counts[len(sorted_counts) // 2]
|
|
p90 = sorted_counts[min(len(sorted_counts) - 1, int(len(sorted_counts) * 0.9))]
|
|
avg = sum(sorted_counts) / len(sorted_counts)
|
|
print(
|
|
" job_count50 "
|
|
f"n={len(sorted_counts)} min={sorted_counts[0]} p50={p50} "
|
|
f"p90={p90} max={sorted_counts[-1]} avg={avg:.1f}"
|
|
)
|
|
|
|
print(" low-byte class counts:", dict(first_byte.most_common()))
|
|
if lane_lookup:
|
|
print(f" matched route records to lane hashes: {matched_handles}/{matched_handles + unmatched_handles}")
|
|
print(" class -> resource categories:")
|
|
for class_id, count in first_byte.most_common():
|
|
categories = dict(class_categories[class_id].most_common())
|
|
speeds = dict(class_speed[class_id].most_common(8))
|
|
flags = dict(class_flags[class_id].most_common(8))
|
|
print(f" {class_id}: categories={categories} speeds={speeds} flags={flags}")
|
|
print_per_route(path, submits, route_results, lane_lookup)
|
|
for index, counter in enumerate(byte_positions):
|
|
print(f" byte{index} counts:", dict(counter.most_common(12)))
|
|
print(" packed metadata patterns:")
|
|
for pattern, count in patterns.most_common(24):
|
|
print(f" {pattern}: {count}")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--traffic-json", type=Path)
|
|
parser.add_argument("logs", nargs="+", type=Path)
|
|
args = parser.parse_args()
|
|
|
|
lane_lookup = load_lane_lookup(args.traffic_json)
|
|
for path in args.logs:
|
|
summarize(path, lane_lookup)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|