mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
feat(hw/sim): distinguish query-only and end-to-end performance cycles in retrieval benchmark
Add explicit separation between query-only cycles (accept→last) and end-to-end cycles (load + write + noise + queries) in hardware retrieval benchmarks. - Add query_only_cycles_per_query, load_write_noise_cycles, end_to_end_cycles metrics - Refactor summarize_query_timings() to use accept_to_last_result_cycles as query-only base - Add build_hardware_performance() to compute end-to-end performance separately - Add current_sim_cycle() helper using cocotb get_sim_time - Update CSV/Markdown outputs and RETRIEVAL_PERF_RESULT log format - Update documentation to clarify cycle-counting methodology - Update tests to cover new performance measurement logic
This commit is contained in:
@@ -11,38 +11,73 @@ if str(HW_SIM_DIR) not in sys.path:
|
||||
|
||||
from benchmarks.retrieval.test_retrieval_benchmark import ( # noqa: E402
|
||||
QueryTiming,
|
||||
build_hardware_performance,
|
||||
summarize_query_timings,
|
||||
write_outputs,
|
||||
)
|
||||
|
||||
|
||||
def test_summarize_query_timings_reports_topk_completion_headline() -> None:
|
||||
def test_summarize_query_timings_uses_query_only_accept_to_last_cycles() -> None:
|
||||
summary = summarize_query_timings([
|
||||
QueryTiming(
|
||||
accept_to_first_result_cycles=10,
|
||||
accept_to_last_result_cycles=14,
|
||||
total_query_cycles=16,
|
||||
total_query_cycles=14,
|
||||
),
|
||||
QueryTiming(
|
||||
accept_to_first_result_cycles=12,
|
||||
accept_to_last_result_cycles=18,
|
||||
total_query_cycles=21,
|
||||
total_query_cycles=18,
|
||||
),
|
||||
])
|
||||
|
||||
assert summary == {
|
||||
"num_queries": 2,
|
||||
"total_query_cycles": 37,
|
||||
"mean_total_query_cycles": 18.5,
|
||||
"min_total_query_cycles": 16,
|
||||
"max_total_query_cycles": 21,
|
||||
"query_only_total_cycles": 32,
|
||||
"query_only_cycles_per_query": 16.0,
|
||||
"query_only_min_cycles": 14,
|
||||
"query_only_max_cycles": 18,
|
||||
"query_only_queries_per_cycle": 2 / 32,
|
||||
"mean_accept_to_first_result_cycles": 11.0,
|
||||
"mean_accept_to_last_result_cycles": 16.0,
|
||||
"total_query_cycles": 32,
|
||||
"mean_total_query_cycles": 16.0,
|
||||
"min_total_query_cycles": 14,
|
||||
"max_total_query_cycles": 18,
|
||||
"cycles_per_query": 16.0,
|
||||
"queries_per_cycle": 2 / 37,
|
||||
"queries_per_cycle": 2 / 32,
|
||||
}
|
||||
|
||||
|
||||
def test_build_hardware_performance_separates_query_and_end_to_end_cycles() -> None:
|
||||
performance = build_hardware_performance(
|
||||
[
|
||||
QueryTiming(
|
||||
accept_to_first_result_cycles=10,
|
||||
accept_to_last_result_cycles=14,
|
||||
total_query_cycles=14,
|
||||
),
|
||||
QueryTiming(
|
||||
accept_to_first_result_cycles=12,
|
||||
accept_to_last_result_cycles=18,
|
||||
total_query_cycles=18,
|
||||
),
|
||||
],
|
||||
load_write_noise_cycles=100,
|
||||
end_to_end_cycles=140,
|
||||
)
|
||||
|
||||
assert performance["query_only_total_cycles"] == 32
|
||||
assert performance["query_only_cycles_per_query"] == 16.0
|
||||
assert performance["query_only_queries_per_cycle"] == 2 / 32
|
||||
assert performance["load_write_noise_cycles"] == 100
|
||||
assert performance["end_to_end_cycles"] == 140
|
||||
assert performance["end_to_end_cycles_per_query"] == 70.0
|
||||
assert performance["end_to_end_queries_per_cycle"] == 2 / 140
|
||||
assert performance["cycles_per_query"] == performance["query_only_cycles_per_query"]
|
||||
assert performance["queries_per_cycle"] == performance["query_only_queries_per_cycle"]
|
||||
|
||||
|
||||
def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> None:
|
||||
result = {
|
||||
"run_id": "test-run",
|
||||
@@ -75,14 +110,23 @@ def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> N
|
||||
},
|
||||
"performance": {
|
||||
"num_queries": 2,
|
||||
"total_query_cycles": 37,
|
||||
"mean_total_query_cycles": 18.5,
|
||||
"min_total_query_cycles": 16,
|
||||
"max_total_query_cycles": 21,
|
||||
"query_only_total_cycles": 32,
|
||||
"query_only_cycles_per_query": 16.0,
|
||||
"query_only_min_cycles": 14,
|
||||
"query_only_max_cycles": 18,
|
||||
"query_only_queries_per_cycle": 2 / 32,
|
||||
"mean_accept_to_first_result_cycles": 11.0,
|
||||
"mean_accept_to_last_result_cycles": 16.0,
|
||||
"load_write_noise_cycles": 100,
|
||||
"end_to_end_cycles": 140,
|
||||
"end_to_end_cycles_per_query": 70.0,
|
||||
"end_to_end_queries_per_cycle": 2 / 140,
|
||||
"total_query_cycles": 32,
|
||||
"mean_total_query_cycles": 16.0,
|
||||
"min_total_query_cycles": 14,
|
||||
"max_total_query_cycles": 18,
|
||||
"cycles_per_query": 16.0,
|
||||
"queries_per_cycle": 2 / 37,
|
||||
"queries_per_cycle": 2 / 32,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -91,12 +135,20 @@ def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> N
|
||||
with (tmp_path / "metrics.csv").open(newline="", encoding="utf-8") as f:
|
||||
row = next(csv.DictReader(f))
|
||||
|
||||
assert row["query_only_cycles_per_query"] == "16.0"
|
||||
assert row["query_only_total_cycles"] == "32"
|
||||
assert row["query_only_queries_per_cycle"] == str(2 / 32)
|
||||
assert row["load_write_noise_cycles"] == "100"
|
||||
assert row["end_to_end_cycles"] == "140"
|
||||
assert row["end_to_end_queries_per_cycle"] == str(2 / 140)
|
||||
assert row["cycles_per_query"] == "16.0"
|
||||
assert row["mean_accept_to_first_result_cycles"] == "11.0"
|
||||
assert row["mean_accept_to_last_result_cycles"] == "16.0"
|
||||
assert row["queries_per_cycle"] == str(2 / 37)
|
||||
assert row["queries_per_cycle"] == str(2 / 32)
|
||||
|
||||
summary = (tmp_path / "summary.md").read_text(encoding="utf-8")
|
||||
assert "## Hardware performance" in summary
|
||||
assert "cycles_per_query" in summary
|
||||
assert "query-only cycles/query" in summary
|
||||
assert "load/write/noise cycles" in summary
|
||||
assert "end-to-end cycles/query" in summary
|
||||
assert "accept_to_last_result_cycles" in summary
|
||||
|
||||
Reference in New Issue
Block a user