from __future__ import annotations import csv import sys from pathlib import Path HW_SIM_DIR = Path(__file__).resolve().parents[1] / "hw" / "sim" if str(HW_SIM_DIR) not in sys.path: sys.path.insert(0, str(HW_SIM_DIR)) from benchmarks.retrieval.test_retrieval_benchmark import ( # noqa: E402 QueryTiming, summarize_query_timings, write_outputs, ) def test_summarize_query_timings_reports_topk_completion_headline() -> None: summary = summarize_query_timings([ QueryTiming( accept_to_first_result_cycles=10, accept_to_last_result_cycles=14, total_query_cycles=16, ), QueryTiming( accept_to_first_result_cycles=12, accept_to_last_result_cycles=18, total_query_cycles=21, ), ]) assert summary == { "num_queries": 2, "total_query_cycles": 37, "mean_total_query_cycles": 18.5, "min_total_query_cycles": 16, "max_total_query_cycles": 21, "mean_accept_to_first_result_cycles": 11.0, "mean_accept_to_last_result_cycles": 16.0, "cycles_per_query": 16.0, "queries_per_cycle": 2 / 37, } def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> None: result = { "run_id": "test-run", "mode": "no_noise", "status": "pass", "params": { "num_rows": 512, "hash_bits": 512, "lanes": 8, "topk_k": 5, "write_noise_en": 0, "write_noise_rate_num": 0, "write_noise_rate_den": 100, }, "dataset": { "num_classes": 10, "positives_per_class": 0, "queries_per_class": 0, "num_queries": 2, "seed": 0, }, "metrics": { "1": { "macro_precision": 1.0, "retrieval_recall": 0.5, "macro_f1": 2 / 3, "recall@k": 1.0, "exact_match_rate": 1.0, } }, "performance": { "num_queries": 2, "total_query_cycles": 37, "mean_total_query_cycles": 18.5, "min_total_query_cycles": 16, "max_total_query_cycles": 21, "mean_accept_to_first_result_cycles": 11.0, "mean_accept_to_last_result_cycles": 16.0, "cycles_per_query": 16.0, "queries_per_cycle": 2 / 37, }, } write_outputs(tmp_path, result) with (tmp_path / "metrics.csv").open(newline="", encoding="utf-8") as f: row = next(csv.DictReader(f)) assert row["cycles_per_query"] == "16.0" assert row["mean_accept_to_first_result_cycles"] == "11.0" assert row["mean_accept_to_last_result_cycles"] == "16.0" assert row["queries_per_cycle"] == str(2 / 37) summary = (tmp_path / "summary.md").read_text(encoding="utf-8") assert "## Hardware performance" in summary assert "cycles_per_query" in summary assert "accept_to_last_result_cycles" in summary