feat(hw/sim): distinguish query-only and end-to-end performance cycles in retrieval benchmark

Add explicit separation between query-only cycles (accept→last) and end-to-end cycles
(load + write + noise + queries) in hardware retrieval benchmarks.

- Add query_only_cycles_per_query, load_write_noise_cycles, end_to_end_cycles metrics
- Refactor summarize_query_timings() to use accept_to_last_result_cycles as query-only base
- Add build_hardware_performance() to compute end-to-end performance separately
- Add current_sim_cycle() helper using cocotb get_sim_time
- Update CSV/Markdown outputs and RETRIEVAL_PERF_RESULT log format
- Update documentation to clarify cycle-counting methodology
- Update tests to cover new performance measurement logic
This commit is contained in:
2026-05-28 13:46:09 +08:00
parent 42d4a9728d
commit 97e53d44f8
4 changed files with 195 additions and 56 deletions

View File

@@ -11,38 +11,73 @@ if str(HW_SIM_DIR) not in sys.path:
from benchmarks.retrieval.test_retrieval_benchmark import ( # noqa: E402
QueryTiming,
build_hardware_performance,
summarize_query_timings,
write_outputs,
)
def test_summarize_query_timings_reports_topk_completion_headline() -> None:
def test_summarize_query_timings_uses_query_only_accept_to_last_cycles() -> None:
summary = summarize_query_timings([
QueryTiming(
accept_to_first_result_cycles=10,
accept_to_last_result_cycles=14,
total_query_cycles=16,
total_query_cycles=14,
),
QueryTiming(
accept_to_first_result_cycles=12,
accept_to_last_result_cycles=18,
total_query_cycles=21,
total_query_cycles=18,
),
])
assert summary == {
"num_queries": 2,
"total_query_cycles": 37,
"mean_total_query_cycles": 18.5,
"min_total_query_cycles": 16,
"max_total_query_cycles": 21,
"query_only_total_cycles": 32,
"query_only_cycles_per_query": 16.0,
"query_only_min_cycles": 14,
"query_only_max_cycles": 18,
"query_only_queries_per_cycle": 2 / 32,
"mean_accept_to_first_result_cycles": 11.0,
"mean_accept_to_last_result_cycles": 16.0,
"total_query_cycles": 32,
"mean_total_query_cycles": 16.0,
"min_total_query_cycles": 14,
"max_total_query_cycles": 18,
"cycles_per_query": 16.0,
"queries_per_cycle": 2 / 37,
"queries_per_cycle": 2 / 32,
}
def test_build_hardware_performance_separates_query_and_end_to_end_cycles() -> None:
performance = build_hardware_performance(
[
QueryTiming(
accept_to_first_result_cycles=10,
accept_to_last_result_cycles=14,
total_query_cycles=14,
),
QueryTiming(
accept_to_first_result_cycles=12,
accept_to_last_result_cycles=18,
total_query_cycles=18,
),
],
load_write_noise_cycles=100,
end_to_end_cycles=140,
)
assert performance["query_only_total_cycles"] == 32
assert performance["query_only_cycles_per_query"] == 16.0
assert performance["query_only_queries_per_cycle"] == 2 / 32
assert performance["load_write_noise_cycles"] == 100
assert performance["end_to_end_cycles"] == 140
assert performance["end_to_end_cycles_per_query"] == 70.0
assert performance["end_to_end_queries_per_cycle"] == 2 / 140
assert performance["cycles_per_query"] == performance["query_only_cycles_per_query"]
assert performance["queries_per_cycle"] == performance["query_only_queries_per_cycle"]
def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> None:
result = {
"run_id": "test-run",
@@ -75,14 +110,23 @@ def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> N
},
"performance": {
"num_queries": 2,
"total_query_cycles": 37,
"mean_total_query_cycles": 18.5,
"min_total_query_cycles": 16,
"max_total_query_cycles": 21,
"query_only_total_cycles": 32,
"query_only_cycles_per_query": 16.0,
"query_only_min_cycles": 14,
"query_only_max_cycles": 18,
"query_only_queries_per_cycle": 2 / 32,
"mean_accept_to_first_result_cycles": 11.0,
"mean_accept_to_last_result_cycles": 16.0,
"load_write_noise_cycles": 100,
"end_to_end_cycles": 140,
"end_to_end_cycles_per_query": 70.0,
"end_to_end_queries_per_cycle": 2 / 140,
"total_query_cycles": 32,
"mean_total_query_cycles": 16.0,
"min_total_query_cycles": 14,
"max_total_query_cycles": 18,
"cycles_per_query": 16.0,
"queries_per_cycle": 2 / 37,
"queries_per_cycle": 2 / 32,
},
}
@@ -91,12 +135,20 @@ def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> N
with (tmp_path / "metrics.csv").open(newline="", encoding="utf-8") as f:
row = next(csv.DictReader(f))
assert row["query_only_cycles_per_query"] == "16.0"
assert row["query_only_total_cycles"] == "32"
assert row["query_only_queries_per_cycle"] == str(2 / 32)
assert row["load_write_noise_cycles"] == "100"
assert row["end_to_end_cycles"] == "140"
assert row["end_to_end_queries_per_cycle"] == str(2 / 140)
assert row["cycles_per_query"] == "16.0"
assert row["mean_accept_to_first_result_cycles"] == "11.0"
assert row["mean_accept_to_last_result_cycles"] == "16.0"
assert row["queries_per_cycle"] == str(2 / 37)
assert row["queries_per_cycle"] == str(2 / 32)
summary = (tmp_path / "summary.md").read_text(encoding="utf-8")
assert "## Hardware performance" in summary
assert "cycles_per_query" in summary
assert "query-only cycles/query" in summary
assert "load/write/noise cycles" in summary
assert "end-to-end cycles/query" in summary
assert "accept_to_last_result_cycles" in summary