SikongJueluo/Mini-Nav
1
0
Fork 0
mirror of https://github.com/SikongJueluo/Mini-Nav.git synced 2026-07-12 20:15:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(hw/sim): distinguish query-only and end-to-end performance cycles in retrieval benchmark

Browse Source 
Add explicit separation between query-only cycles (accept→last) and end-to-end cycles
(load + write + noise + queries) in hardware retrieval benchmarks.

- Add query_only_cycles_per_query, load_write_noise_cycles, end_to_end_cycles metrics
- Refactor summarize_query_timings() to use accept_to_last_result_cycles as query-only base
- Add build_hardware_performance() to compute end-to-end performance separately
- Add current_sim_cycle() helper using cocotb get_sim_time
- Update CSV/Markdown outputs and RETRIEVAL_PERF_RESULT log format
- Update documentation to clarify cycle-counting methodology
- Update tests to cover new performance measurement logic
This commit is contained in:
SikongJueluo
2026-05-28 13:46:09 +08:00
parent 42d4a9728d
commit 97e53d44f8
4 changed files with 195 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py
View File

@@ -10,6 +10,7 @@ from pathlib import Path
import cocotb
import numpy as np
from cocotb.clock import Clock
from cocotb.utils import get_sim_time
from model.ref_model import (
match_topk,
@@ -32,6 +33,7 @@ DEFAULT_ROW_FLIP_BITS = 16
DEFAULT_QUERY_FLIP_BITS = 16
DEFAULT_SEED = 20260522
BENCHMARK_KS = (1, 5)
SIM_CLOCK_PERIOD_NS = 10
@dataclass(frozen=True)
@@ -214,6 +216,11 @@ def summarize_query_timings(timings: list[QueryTiming]) -> dict[str, float]:
if not timings:
return {
"num_queries": 0,
"query_only_total_cycles": 0,
"query_only_cycles_per_query": 0.0,
"query_only_min_cycles": 0,
"query_only_max_cycles": 0,
"query_only_queries_per_cycle": 0.0,
"total_query_cycles": 0,
"mean_total_query_cycles": 0.0,
"min_total_query_cycles": 0,
@@ -224,24 +231,57 @@ def summarize_query_timings(timings: list[QueryTiming]) -> dict[str, float]:
"queries_per_cycle": 0.0,
}
total_cycles = sum(t.total_query_cycles for t in timings)
total_cycles = sum(t.accept_to_last_result_cycles for t in timings)
total_first = sum(t.accept_to_first_result_cycles for t in timings)
total_last = sum(t.accept_to_last_result_cycles for t in timings)
count = len(timings)
mean_last = total_last / float(count)
queries_per_cycle = count / float(total_cycles) if total_cycles > 0 else 0.0
return {
"num_queries": count,
"query_only_total_cycles": total_cycles,
"query_only_cycles_per_query": mean_last,
"query_only_min_cycles": min(t.accept_to_last_result_cycles for t in timings),
"query_only_max_cycles": max(t.accept_to_last_result_cycles for t in timings),
"query_only_queries_per_cycle": queries_per_cycle,
# Backward-compatible aliases: query-only, not end-to-end.
"total_query_cycles": total_cycles,
"mean_total_query_cycles": total_cycles / float(count),
"min_total_query_cycles": min(t.total_query_cycles for t in timings),
"max_total_query_cycles": max(t.total_query_cycles for t in timings),
"min_total_query_cycles": min(t.accept_to_last_result_cycles for t in timings),
"max_total_query_cycles": max(t.accept_to_last_result_cycles for t in timings),
"mean_accept_to_first_result_cycles": total_first / float(count),
"mean_accept_to_last_result_cycles": mean_last,
"cycles_per_query": mean_last,
"queries_per_cycle": count / float(total_cycles) if total_cycles > 0 else 0.0,
"queries_per_cycle": queries_per_cycle,
}
def build_hardware_performance(
timings: list[QueryTiming],
*,
load_write_noise_cycles: int,
end_to_end_cycles: int,
) -> dict[str, float]:
performance = summarize_query_timings(timings)
num_queries = int(performance["num_queries"])
performance.update({
"load_write_noise_cycles": int(load_write_noise_cycles),
"end_to_end_cycles": int(end_to_end_cycles),
"end_to_end_cycles_per_query": (
float(end_to_end_cycles) / float(num_queries) if num_queries > 0 else 0.0
),
"end_to_end_queries_per_cycle": (
float(num_queries) / float(end_to_end_cycles) if end_to_end_cycles > 0 else 0.0
),
})
return performance
def current_sim_cycle() -> int:
"""Return the current benchmark clock cycle from simulator time."""
return int(get_sim_time("ns") // SIM_CLOCK_PERIOD_NS)
def output_dir_for(mode: str) -> Path:
run_id = os.environ.get("CAM_RETRIEVAL_RUN_ID")
if not run_id:
@@ -264,7 +304,11 @@ def write_outputs(out_dir: Path, result: dict) -> None:
"write_noise_en", "write_noise_rate_num",
"write_noise_rate_den",
"num_queries", "k", "macro_precision", "retrieval_recall", "macro_f1",
"recall@k", "exact_match_rate", "cycles_per_query",
"recall@k", "exact_match_rate",
"query_only_cycles_per_query", "query_only_total_cycles",
"query_only_queries_per_cycle", "load_write_noise_cycles",
"end_to_end_cycles", "end_to_end_cycles_per_query",
"end_to_end_queries_per_cycle", "cycles_per_query",
"mean_accept_to_first_result_cycles", "mean_accept_to_last_result_cycles",
"mean_total_query_cycles", "total_query_cycles", "queries_per_cycle", "status",
]
@@ -289,6 +333,13 @@ def write_outputs(out_dir: Path, result: dict) -> None:
"macro_f1": metrics["macro_f1"],
"recall@k": metrics["recall@k"],
"exact_match_rate": metrics["exact_match_rate"],
"query_only_cycles_per_query": result.get("performance", {}).get("query_only_cycles_per_query", ""),
"query_only_total_cycles": result.get("performance", {}).get("query_only_total_cycles", ""),
"query_only_queries_per_cycle": result.get("performance", {}).get("query_only_queries_per_cycle", ""),
"load_write_noise_cycles": result.get("performance", {}).get("load_write_noise_cycles", ""),
"end_to_end_cycles": result.get("performance", {}).get("end_to_end_cycles", ""),
"end_to_end_cycles_per_query": result.get("performance", {}).get("end_to_end_cycles_per_query", ""),
"end_to_end_queries_per_cycle": result.get("performance", {}).get("end_to_end_queries_per_cycle", ""),
"cycles_per_query": result.get("performance", {}).get("cycles_per_query", ""),
"mean_accept_to_first_result_cycles": result.get("performance", {}).get(
"mean_accept_to_first_result_cycles", "",
@@ -313,11 +364,18 @@ def write_outputs(out_dir: Path, result: dict) -> None:
"",
"## Hardware performance",
"",
f"- cycles_per_query: `{result.get('performance', {}).get('cycles_per_query', '')}`",
f"- query-only cycles/query: `{result.get('performance', {}).get('query_only_cycles_per_query', '')}`",
f"- query-only total cycles: `{result.get('performance', {}).get('query_only_total_cycles', '')}`",
f"- query-only queries/cycle: `{result.get('performance', {}).get('query_only_queries_per_cycle', '')}`",
f"- load/write/noise cycles: `{result.get('performance', {}).get('load_write_noise_cycles', '')}`",
f"- end-to-end cycles: `{result.get('performance', {}).get('end_to_end_cycles', '')}`",
f"- end-to-end cycles/query: `{result.get('performance', {}).get('end_to_end_cycles_per_query', '')}`",
f"- end-to-end queries/cycle: `{result.get('performance', {}).get('end_to_end_queries_per_cycle', '')}`",
f"- cycles_per_query (compat, query-only): `{result.get('performance', {}).get('cycles_per_query', '')}`",
f"- accept_to_first_result_cycles: `{result.get('performance', {}).get('mean_accept_to_first_result_cycles', '')}`",
f"- accept_to_last_result_cycles: `{result.get('performance', {}).get('mean_accept_to_last_result_cycles', '')}`",
f"- total_query_cycles: `{result.get('performance', {}).get('total_query_cycles', '')}`",
f"- queries_per_cycle: `{result.get('performance', {}).get('queries_per_cycle', '')}`",
f"- total_query_cycles (compat, query-only): `{result.get('performance', {}).get('total_query_cycles', '')}`",
f"- queries_per_cycle (compat, query-only): `{result.get('performance', {}).get('queries_per_cycle', '')}`",
"",
"## Retrieval quality",
"",
@@ -338,7 +396,7 @@ def write_outputs(out_dir: Path, result: dict) -> None:
@cocotb.test()
async def cam_retrieval_benchmark(dut):
cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start())
cocotb.start_soon(Clock(dut.clk, SIM_CLOCK_PERIOD_NS, unit="ns").start())
await reset_dut(dut)
num_rows = dut_num_rows(dut)
@@ -358,7 +416,11 @@ async def cam_retrieval_benchmark(dut):
dataset = load_retrieval_dataset_npz(dataset_path)
if len(dataset.rows) != num_rows:
raise AssertionError(f"artifact row count {len(dataset.rows)} must equal DUT NUM_ROWS {num_rows}")
benchmark_start_cycle = current_sim_cycle()
load_write_noise_start_cycle = current_sim_cycle()
await write_rows(dut, dataset.rows)
load_write_noise_cycles = current_sim_cycle() - load_write_noise_start_cycle
accumulators = {k: MetricAccumulator() for k in BENCHMARK_KS}
timings: list[QueryTiming] = []
@@ -380,6 +442,8 @@ async def cam_retrieval_benchmark(dut):
label_hit = query_label in retrieved_labels
accumulators[k] = accumulators[k].add(precision, recall, f1, label_hit, exact)
end_to_end_cycles = current_sim_cycle() - benchmark_start_cycle
run_id = os.environ.get("CAM_RETRIEVAL_RUN_ID") or f"{datetime.now().strftime('%Y-%m-%d-%H%M%S')}-{mode}"
result = {
"run_id": run_id,
@@ -402,7 +466,11 @@ async def cam_retrieval_benchmark(dut):
"seed": dataset.seed,
},
"metrics": {str(k): accumulators[k].as_dict() for k in BENCHMARK_KS},
"performance": summarize_query_timings(timings),
"performance": build_hardware_performance(
timings,
load_write_noise_cycles=load_write_noise_cycles,
end_to_end_cycles=end_to_end_cycles,
),
}
out_dir = output_dir_for(mode)
@@ -419,11 +487,22 @@ async def cam_retrieval_benchmark(dut):
performance = result["performance"]
dut._log.info(
"RETRIEVAL_PERF_RESULT mode=%s num_queries=%d cycles_per_query=%.6f "
"RETRIEVAL_PERF_RESULT mode=%s num_queries=%d query_only_cycles_per_query=%.6f "
"query_only_total_cycles=%d query_only_queries_per_cycle=%.9f "
"load_write_noise_cycles=%d end_to_end_cycles=%d "
"end_to_end_cycles_per_query=%.6f end_to_end_queries_per_cycle=%.9f "
"cycles_per_query=%.6f "
"accept_to_first_result_cycles=%.6f accept_to_last_result_cycles=%.6f "
"total_query_cycles=%d queries_per_cycle=%.9f status=pass output_dir=%s",
mode,
performance["num_queries"],
performance["query_only_cycles_per_query"],
performance["query_only_total_cycles"],
performance["query_only_queries_per_cycle"],
performance["load_write_noise_cycles"],
performance["end_to_end_cycles"],
performance["end_to_end_cycles_per_query"],
performance["end_to_end_queries_per_cycle"],
performance["cycles_per_query"],
performance["mean_accept_to_first_result_cycles"],
performance["mean_accept_to_last_result_cycles"],