mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
feat: add hardware retrieval cycle performance measurement
Add cycle-level performance measurement for hardware CAM retrieval benchmarks to complement existing quality metrics. - Add query_topk_once_with_latency with accept→first/last cycle timing - Add QueryTiming dataclass and summarize_query_timings helper - Integrate cycle performance into benchmark outputs (CSV + Markdown) - Log RETRIEVAL_PERF_RESULT with cycles/query and queries/cycle - Update experiment docs with hardware cycle performance section - Add unit tests for summarize_query_timings and output writers
This commit is contained in:
102
tests/test_retrieval_benchmark_performance.py
Normal file
102
tests/test_retrieval_benchmark_performance.py
Normal file
@@ -0,0 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
HW_SIM_DIR = Path(__file__).resolve().parents[1] / "hw" / "sim"
|
||||
if str(HW_SIM_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(HW_SIM_DIR))
|
||||
|
||||
from benchmarks.retrieval.test_retrieval_benchmark import ( # noqa: E402
|
||||
QueryTiming,
|
||||
summarize_query_timings,
|
||||
write_outputs,
|
||||
)
|
||||
|
||||
|
||||
def test_summarize_query_timings_reports_topk_completion_headline() -> None:
|
||||
summary = summarize_query_timings([
|
||||
QueryTiming(
|
||||
accept_to_first_result_cycles=10,
|
||||
accept_to_last_result_cycles=14,
|
||||
total_query_cycles=16,
|
||||
),
|
||||
QueryTiming(
|
||||
accept_to_first_result_cycles=12,
|
||||
accept_to_last_result_cycles=18,
|
||||
total_query_cycles=21,
|
||||
),
|
||||
])
|
||||
|
||||
assert summary == {
|
||||
"num_queries": 2,
|
||||
"total_query_cycles": 37,
|
||||
"mean_total_query_cycles": 18.5,
|
||||
"min_total_query_cycles": 16,
|
||||
"max_total_query_cycles": 21,
|
||||
"mean_accept_to_first_result_cycles": 11.0,
|
||||
"mean_accept_to_last_result_cycles": 16.0,
|
||||
"cycles_per_query": 16.0,
|
||||
"queries_per_cycle": 2 / 37,
|
||||
}
|
||||
|
||||
|
||||
def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> None:
|
||||
result = {
|
||||
"run_id": "test-run",
|
||||
"mode": "no_noise",
|
||||
"status": "pass",
|
||||
"params": {
|
||||
"num_rows": 512,
|
||||
"hash_bits": 512,
|
||||
"lanes": 8,
|
||||
"topk_k": 5,
|
||||
"write_noise_en": 0,
|
||||
"write_noise_rate_num": 0,
|
||||
"write_noise_rate_den": 100,
|
||||
},
|
||||
"dataset": {
|
||||
"num_classes": 10,
|
||||
"positives_per_class": 0,
|
||||
"queries_per_class": 0,
|
||||
"num_queries": 2,
|
||||
"seed": 0,
|
||||
},
|
||||
"metrics": {
|
||||
"1": {
|
||||
"macro_precision": 1.0,
|
||||
"retrieval_recall": 0.5,
|
||||
"macro_f1": 2 / 3,
|
||||
"recall@k": 1.0,
|
||||
"exact_match_rate": 1.0,
|
||||
}
|
||||
},
|
||||
"performance": {
|
||||
"num_queries": 2,
|
||||
"total_query_cycles": 37,
|
||||
"mean_total_query_cycles": 18.5,
|
||||
"min_total_query_cycles": 16,
|
||||
"max_total_query_cycles": 21,
|
||||
"mean_accept_to_first_result_cycles": 11.0,
|
||||
"mean_accept_to_last_result_cycles": 16.0,
|
||||
"cycles_per_query": 16.0,
|
||||
"queries_per_cycle": 2 / 37,
|
||||
},
|
||||
}
|
||||
|
||||
write_outputs(tmp_path, result)
|
||||
|
||||
with (tmp_path / "metrics.csv").open(newline="", encoding="utf-8") as f:
|
||||
row = next(csv.DictReader(f))
|
||||
|
||||
assert row["cycles_per_query"] == "16.0"
|
||||
assert row["mean_accept_to_first_result_cycles"] == "11.0"
|
||||
assert row["mean_accept_to_last_result_cycles"] == "16.0"
|
||||
assert row["queries_per_cycle"] == str(2 / 37)
|
||||
|
||||
summary = (tmp_path / "summary.md").read_text(encoding="utf-8")
|
||||
assert "## Hardware performance" in summary
|
||||
assert "cycles_per_query" in summary
|
||||
assert "accept_to_last_result_cycles" in summary
|
||||
Reference in New Issue
Block a user