feat: add hardware retrieval cycle performance measurement

Add cycle-level performance measurement for hardware CAM retrieval benchmarks
to complement existing quality metrics.

- Add query_topk_once_with_latency with accept→first/last cycle timing
- Add QueryTiming dataclass and summarize_query_timings helper
- Integrate cycle performance into benchmark outputs (CSV + Markdown)
- Log RETRIEVAL_PERF_RESULT with cycles/query and queries/cycle
- Update experiment docs with hardware cycle performance section
- Add unit tests for summarize_query_timings and output writers
This commit is contained in:
2026-05-27 20:28:50 +08:00
parent 715a2c2ed6
commit 42d4a9728d
4 changed files with 304 additions and 20 deletions

View File

@@ -0,0 +1,102 @@
from __future__ import annotations
import csv
import sys
from pathlib import Path
HW_SIM_DIR = Path(__file__).resolve().parents[1] / "hw" / "sim"
if str(HW_SIM_DIR) not in sys.path:
sys.path.insert(0, str(HW_SIM_DIR))
from benchmarks.retrieval.test_retrieval_benchmark import ( # noqa: E402
QueryTiming,
summarize_query_timings,
write_outputs,
)
def test_summarize_query_timings_reports_topk_completion_headline() -> None:
summary = summarize_query_timings([
QueryTiming(
accept_to_first_result_cycles=10,
accept_to_last_result_cycles=14,
total_query_cycles=16,
),
QueryTiming(
accept_to_first_result_cycles=12,
accept_to_last_result_cycles=18,
total_query_cycles=21,
),
])
assert summary == {
"num_queries": 2,
"total_query_cycles": 37,
"mean_total_query_cycles": 18.5,
"min_total_query_cycles": 16,
"max_total_query_cycles": 21,
"mean_accept_to_first_result_cycles": 11.0,
"mean_accept_to_last_result_cycles": 16.0,
"cycles_per_query": 16.0,
"queries_per_cycle": 2 / 37,
}
def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> None:
result = {
"run_id": "test-run",
"mode": "no_noise",
"status": "pass",
"params": {
"num_rows": 512,
"hash_bits": 512,
"lanes": 8,
"topk_k": 5,
"write_noise_en": 0,
"write_noise_rate_num": 0,
"write_noise_rate_den": 100,
},
"dataset": {
"num_classes": 10,
"positives_per_class": 0,
"queries_per_class": 0,
"num_queries": 2,
"seed": 0,
},
"metrics": {
"1": {
"macro_precision": 1.0,
"retrieval_recall": 0.5,
"macro_f1": 2 / 3,
"recall@k": 1.0,
"exact_match_rate": 1.0,
}
},
"performance": {
"num_queries": 2,
"total_query_cycles": 37,
"mean_total_query_cycles": 18.5,
"min_total_query_cycles": 16,
"max_total_query_cycles": 21,
"mean_accept_to_first_result_cycles": 11.0,
"mean_accept_to_last_result_cycles": 16.0,
"cycles_per_query": 16.0,
"queries_per_cycle": 2 / 37,
},
}
write_outputs(tmp_path, result)
with (tmp_path / "metrics.csv").open(newline="", encoding="utf-8") as f:
row = next(csv.DictReader(f))
assert row["cycles_per_query"] == "16.0"
assert row["mean_accept_to_first_result_cycles"] == "11.0"
assert row["mean_accept_to_last_result_cycles"] == "16.0"
assert row["queries_per_cycle"] == str(2 / 37)
summary = (tmp_path / "summary.md").read_text(encoding="utf-8")
assert "## Hardware performance" in summary
assert "cycles_per_query" in summary
assert "accept_to_last_result_cycles" in summary