SikongJueluo/Mini-Nav
1
0
Fork 0
mirror of https://github.com/SikongJueluo/Mini-Nav.git synced 2026-07-12 20:15:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add hardware retrieval cycle performance measurement

Browse Source 
Add cycle-level performance measurement for hardware CAM retrieval benchmarks
to complement existing quality metrics.

- Add query_topk_once_with_latency with accept→first/last cycle timing
- Add QueryTiming dataclass and summarize_query_timings helper
- Integrate cycle performance into benchmark outputs (CSV + Markdown)
- Log RETRIEVAL_PERF_RESULT with cycles/query and queries/cycle
- Update experiment docs with hardware cycle performance section
- Add unit tests for summarize_query_timings and output writers
This commit is contained in:
SikongJueluo
2026-05-27 20:28:50 +08:00
parent 715a2c2ed6
commit 42d4a9728d
4 changed files with 304 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py
View File

@@ -20,7 +20,7 @@ from tests.top.utils import (
dut_lanes,
dut_num_rows,
get_param,
query_topk_once,
query_topk_once_with_latency,
reset_dut,
write_rows,
)
@@ -46,6 +46,13 @@ class RetrievalDataset:
seed: int
@dataclass(frozen=True)
class QueryTiming:
accept_to_first_result_cycles: int
accept_to_last_result_cycles: int
total_query_cycles: int
@dataclass(frozen=True)
class MetricAccumulator:
precision_sum: float = 0.0
@@ -203,6 +210,38 @@ def mode_from_params(write_noise_en: int) -> str:
return "no_noise"
def summarize_query_timings(timings: list[QueryTiming]) -> dict[str, float]:
if not timings:
return {
"num_queries": 0,
"total_query_cycles": 0,
"mean_total_query_cycles": 0.0,
"min_total_query_cycles": 0,
"max_total_query_cycles": 0,
"mean_accept_to_first_result_cycles": 0.0,
"mean_accept_to_last_result_cycles": 0.0,
"cycles_per_query": 0.0,
"queries_per_cycle": 0.0,
}
total_cycles = sum(t.total_query_cycles for t in timings)
total_first = sum(t.accept_to_first_result_cycles for t in timings)
total_last = sum(t.accept_to_last_result_cycles for t in timings)
count = len(timings)
mean_last = total_last / float(count)
return {
"num_queries": count,
"total_query_cycles": total_cycles,
"mean_total_query_cycles": total_cycles / float(count),
"min_total_query_cycles": min(t.total_query_cycles for t in timings),
"max_total_query_cycles": max(t.total_query_cycles for t in timings),
"mean_accept_to_first_result_cycles": total_first / float(count),
"mean_accept_to_last_result_cycles": mean_last,
"cycles_per_query": mean_last,
"queries_per_cycle": count / float(total_cycles) if total_cycles > 0 else 0.0,
}
def output_dir_for(mode: str) -> Path:
run_id = os.environ.get("CAM_RETRIEVAL_RUN_ID")
if not run_id:
@@ -225,7 +264,9 @@ def write_outputs(out_dir: Path, result: dict) -> None:
"write_noise_en", "write_noise_rate_num",
"write_noise_rate_den",
"num_queries", "k", "macro_precision", "retrieval_recall", "macro_f1",
"recall@k", "exact_match_rate", "status",
"recall@k", "exact_match_rate", "cycles_per_query",
"mean_accept_to_first_result_cycles", "mean_accept_to_last_result_cycles",
"mean_total_query_cycles", "total_query_cycles", "queries_per_cycle", "status",
]
with metrics_csv.open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
@@ -248,6 +289,16 @@ def write_outputs(out_dir: Path, result: dict) -> None:
"macro_f1": metrics["macro_f1"],
"recall@k": metrics["recall@k"],
"exact_match_rate": metrics["exact_match_rate"],
"cycles_per_query": result.get("performance", {}).get("cycles_per_query", ""),
"mean_accept_to_first_result_cycles": result.get("performance", {}).get(
"mean_accept_to_first_result_cycles", "",
),
"mean_accept_to_last_result_cycles": result.get("performance", {}).get(
"mean_accept_to_last_result_cycles", "",
),
"mean_total_query_cycles": result.get("performance", {}).get("mean_total_query_cycles", ""),
"total_query_cycles": result.get("performance", {}).get("total_query_cycles", ""),
"queries_per_cycle": result.get("performance", {}).get("queries_per_cycle", ""),
"status": result["status"],
}
writer.writerow(row)
@@ -260,6 +311,16 @@ def write_outputs(out_dir: Path, result: dict) -> None:
f"- status: `{result['status']}`",
f"- num_queries: `{result['dataset']['num_queries']}`",
"",
"## Hardware performance",
"",
f"- cycles_per_query: `{result.get('performance', {}).get('cycles_per_query', '')}`",
f"- accept_to_first_result_cycles: `{result.get('performance', {}).get('mean_accept_to_first_result_cycles', '')}`",
f"- accept_to_last_result_cycles: `{result.get('performance', {}).get('mean_accept_to_last_result_cycles', '')}`",
f"- total_query_cycles: `{result.get('performance', {}).get('total_query_cycles', '')}`",
f"- queries_per_cycle: `{result.get('performance', {}).get('queries_per_cycle', '')}`",
"",
"## Retrieval quality",
"",
"| k | macro_precision | retrieval_recall | macro_f1 | recall@k | exact_match_rate |",
"|---:|---:|---:|---:|---:|---:|",
]
@@ -300,9 +361,11 @@ async def cam_retrieval_benchmark(dut):
await write_rows(dut, dataset.rows)
accumulators = {k: MetricAccumulator() for k in BENCHMARK_KS}
timings: list[QueryTiming] = []
for query, query_label in zip(dataset.queries, dataset.query_labels):
beats, _, _, _ = await query_topk_once(dut, query)
beats, _, _, _, timing = await query_topk_once_with_latency(dut, query)
timings.append(QueryTiming(**timing))
if len(beats) < max(BENCHMARK_KS):
raise AssertionError(f"Expected at least {max(BENCHMARK_KS)} Top-K beats, got {len(beats)}")
@@ -339,6 +402,7 @@ async def cam_retrieval_benchmark(dut):
"seed": dataset.seed,
},
"metrics": {str(k): accumulators[k].as_dict() for k in BENCHMARK_KS},
"performance": summarize_query_timings(timings),
}
out_dir = output_dir_for(mode)
@@ -353,6 +417,21 @@ async def cam_retrieval_benchmark(dut):
str(out_dir.relative_to(_project_root())),
)
performance = result["performance"]
dut._log.info(
"RETRIEVAL_PERF_RESULT mode=%s num_queries=%d cycles_per_query=%.6f "
"accept_to_first_result_cycles=%.6f accept_to_last_result_cycles=%.6f "
"total_query_cycles=%d queries_per_cycle=%.9f status=pass output_dir=%s",
mode,
performance["num_queries"],
performance["cycles_per_query"],
performance["mean_accept_to_first_result_cycles"],
performance["mean_accept_to_last_result_cycles"],
performance["total_query_cycles"],
performance["queries_per_cycle"],
str(out_dir.relative_to(_project_root())),
)
if write_noise_en == 0:
assert result["metrics"]["5"]["exact_match_rate"] == 1.0, (
f"Expected perfect exact match with no noise, got "

66
hw/sim/tests/top/utils.py
View File

@@ -15,7 +15,7 @@ CAM 顶层测试的共享辅助函数。
from __future__ import annotations
import numpy as np
from cocotb.triggers import RisingEdge
from cocotb.triggers import ReadOnly, RisingEdge, Timer
from model.ref_model import ( # noqa: E402
match_top1,
unpack_score_debug_flat,
@@ -202,28 +202,73 @@ async def query_topk_once(dut, query, timeout_cycles=None):
- beats: [(rank, row, score, last), ...]
- score_debug: np.ndarray 或 None（SIM_DEBUG 模式）
"""
beats, top1_index, top1_score, score_debug, _ = await query_topk_once_with_latency(
dut, query, timeout_cycles=timeout_cycles,
)
return beats, top1_index, top1_score, score_debug
async def query_topk_once_with_latency(dut, query, timeout_cycles=None):
"""发起一次查询、收集完整 Top-K 结果流，并返回周期计数。
返回：(beats, top1_index, top1_score, score_debug, timing)
``timing`` 字段：
- accept_to_first_result_cycles: query 接受到首个 result_valid beat
- accept_to_last_result_cycles: query 接受到 result_last beat（Top-K 完成）
- total_query_cycles: 从拉高 query_valid 到 Top-K 完成的总上升沿数
``query_ready`` 是组合信号，握手周期在上升沿前采样；结果信号在
ReadOnly settled phase 采样，避免重新引入 query_ready 采样时序问题。
"""
await wait_idle(dut)
dut.query_hash.value = int(query)
# 等待 query_ready 为高（DUT 已就绪），避免组合逻辑下降沿导致的
# valid&&ready 握手丢失问题
# valid&&ready 握手丢失问题。
while not int(dut.query_ready.value):
await RisingEdge(dut.clk)
# assert query_valid 覆盖一个上升沿完成握手
edge_count = 0
dut.query_valid.value = 1
dut.result_ready.value = 1
await RisingEdge(dut.clk)
edge_count += 1
q_ready = int(dut.query_ready.value)
assert q_ready, "Query accept handshake was missed despite query_ready pre-wait"
accept_edge = edge_count
dut.query_valid.value = 0
# 若调用者未指定超时，根据 DUT 参数动态估算
if timeout_cycles is None:
timeout_cycles = dut_query_timeout_cycles(dut)
# 消费完整串行结果流
beats = await collect_topk(dut, timeout_cycles=timeout_cycles)
beats = []
first_result_edge = None
last_result_edge = None
for _ in range(timeout_cycles):
await RisingEdge(dut.clk)
edge_count += 1
await ReadOnly()
if int(dut.result_valid.value):
if first_result_edge is None:
first_result_edge = edge_count
rank = int(dut.result_rank.value)
row = int(dut.result_row.value)
score = int(dut.result_score.value)
last = int(dut.result_last.value)
beats.append((rank, row, score, last))
if last:
last_result_edge = edge_count
await Timer(1, units="step")
break
await Timer(1, units="step")
if first_result_edge is None or last_result_edge is None:
raise AssertionError("Top-K result stream did not finish")
# score_debug 在查询完成后可用（需 SIM_DEBUG 编译）
num_rows = dut_num_rows(dut)
score_bits = dut_score_bits(dut)
score_debug = None
@@ -234,7 +279,12 @@ async def query_topk_once(dut, query, timeout_cycles=None):
score_bits,
)
return beats, beats[0][1], beats[0][2], score_debug
timing = {
"accept_to_first_result_cycles": int(first_result_edge - accept_edge),
"accept_to_last_result_cycles": int(last_result_edge - accept_edge),
"total_query_cycles": int(edge_count),
}
return beats, beats[0][1], beats[0][2], score_debug, timing
async def query_once(dut, query, timeout_cycles=None):