mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
SikongJueluo
42d4a9728d
Add cycle-level performance measurement for hardware CAM retrieval benchmarks to complement existing quality metrics. - Add query_topk_once_with_latency with accept→first/last cycle timing - Add QueryTiming dataclass and summarize_query_timings helper - Integrate cycle performance into benchmark outputs (CSV + Markdown) - Log RETRIEVAL_PERF_RESULT with cycles/query and queries/cycle - Update experiment docs with hardware cycle performance section - Add unit tests for summarize_query_timings and output writers
103 lines
3.0 KiB
Python
103 lines
3.0 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
HW_SIM_DIR = Path(__file__).resolve().parents[1] / "hw" / "sim"
|
|
if str(HW_SIM_DIR) not in sys.path:
|
|
sys.path.insert(0, str(HW_SIM_DIR))
|
|
|
|
from benchmarks.retrieval.test_retrieval_benchmark import ( # noqa: E402
|
|
QueryTiming,
|
|
summarize_query_timings,
|
|
write_outputs,
|
|
)
|
|
|
|
|
|
def test_summarize_query_timings_reports_topk_completion_headline() -> None:
|
|
summary = summarize_query_timings([
|
|
QueryTiming(
|
|
accept_to_first_result_cycles=10,
|
|
accept_to_last_result_cycles=14,
|
|
total_query_cycles=16,
|
|
),
|
|
QueryTiming(
|
|
accept_to_first_result_cycles=12,
|
|
accept_to_last_result_cycles=18,
|
|
total_query_cycles=21,
|
|
),
|
|
])
|
|
|
|
assert summary == {
|
|
"num_queries": 2,
|
|
"total_query_cycles": 37,
|
|
"mean_total_query_cycles": 18.5,
|
|
"min_total_query_cycles": 16,
|
|
"max_total_query_cycles": 21,
|
|
"mean_accept_to_first_result_cycles": 11.0,
|
|
"mean_accept_to_last_result_cycles": 16.0,
|
|
"cycles_per_query": 16.0,
|
|
"queries_per_cycle": 2 / 37,
|
|
}
|
|
|
|
|
|
def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> None:
|
|
result = {
|
|
"run_id": "test-run",
|
|
"mode": "no_noise",
|
|
"status": "pass",
|
|
"params": {
|
|
"num_rows": 512,
|
|
"hash_bits": 512,
|
|
"lanes": 8,
|
|
"topk_k": 5,
|
|
"write_noise_en": 0,
|
|
"write_noise_rate_num": 0,
|
|
"write_noise_rate_den": 100,
|
|
},
|
|
"dataset": {
|
|
"num_classes": 10,
|
|
"positives_per_class": 0,
|
|
"queries_per_class": 0,
|
|
"num_queries": 2,
|
|
"seed": 0,
|
|
},
|
|
"metrics": {
|
|
"1": {
|
|
"macro_precision": 1.0,
|
|
"retrieval_recall": 0.5,
|
|
"macro_f1": 2 / 3,
|
|
"recall@k": 1.0,
|
|
"exact_match_rate": 1.0,
|
|
}
|
|
},
|
|
"performance": {
|
|
"num_queries": 2,
|
|
"total_query_cycles": 37,
|
|
"mean_total_query_cycles": 18.5,
|
|
"min_total_query_cycles": 16,
|
|
"max_total_query_cycles": 21,
|
|
"mean_accept_to_first_result_cycles": 11.0,
|
|
"mean_accept_to_last_result_cycles": 16.0,
|
|
"cycles_per_query": 16.0,
|
|
"queries_per_cycle": 2 / 37,
|
|
},
|
|
}
|
|
|
|
write_outputs(tmp_path, result)
|
|
|
|
with (tmp_path / "metrics.csv").open(newline="", encoding="utf-8") as f:
|
|
row = next(csv.DictReader(f))
|
|
|
|
assert row["cycles_per_query"] == "16.0"
|
|
assert row["mean_accept_to_first_result_cycles"] == "11.0"
|
|
assert row["mean_accept_to_last_result_cycles"] == "16.0"
|
|
assert row["queries_per_cycle"] == str(2 / 37)
|
|
|
|
summary = (tmp_path / "summary.md").read_text(encoding="utf-8")
|
|
assert "## Hardware performance" in summary
|
|
assert "cycles_per_query" in summary
|
|
assert "accept_to_last_result_cycles" in summary
|