From 42d4a9728d851e5e76229f29a08923a55633b010 Mon Sep 17 00:00:00 2001 From: SikongJueluo Date: Wed, 27 May 2026 20:28:50 +0800 Subject: [PATCH] feat: add hardware retrieval cycle performance measurement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add cycle-level performance measurement for hardware CAM retrieval benchmarks to complement existing quality metrics. - Add query_topk_once_with_latency with accept→first/last cycle timing - Add QueryTiming dataclass and summarize_query_timings helper - Integrate cycle performance into benchmark outputs (CSV + Markdown) - Log RETRIEVAL_PERF_RESULT with cycles/query and queries/cycle - Update experiment docs with hardware cycle performance section - Add unit tests for summarize_query_timings and output writers --- docs/exps/sw_hw_cam_retrieval_benchmark.md | 71 ++++++++++-- .../retrieval/test_retrieval_benchmark.py | 85 ++++++++++++++- hw/sim/tests/top/utils.py | 66 ++++++++++-- tests/test_retrieval_benchmark_performance.py | 102 ++++++++++++++++++ 4 files changed, 304 insertions(+), 20 deletions(-) create mode 100644 tests/test_retrieval_benchmark_performance.py diff --git a/docs/exps/sw_hw_cam_retrieval_benchmark.md b/docs/exps/sw_hw_cam_retrieval_benchmark.md index 06ac2ba..93365fd 100644 --- a/docs/exps/sw_hw_cam_retrieval_benchmark.md +++ b/docs/exps/sw_hw_cam_retrieval_benchmark.md @@ -1,8 +1,8 @@ # 软件/硬件 CAM 检索基准实验总结 **日期**：2026-05-27 -**工作区**：`.workspace/feat_sw_retrieval_benchmark` -**目标**：对比同一组 CAM 检索数据在硬件仿真与软件汉明距离检索下的检索质量，并记录软件检索速度基线。 +**工作区**：`/home/sikongjueluo/Projects/Mini-Nav` +**目标**：对比同一组 CAM 检索数据在硬件仿真与软件汉明距离检索下的检索质量，并记录软件检索速度基线与硬件仿真周期性能。 ## 1. 实验配置 @@ -37,6 +37,12 @@ just remote "python scripts/sw_retrieval_benchmark.py --dataset outputs/cam_retr 硬件噪声扫描数据来自远端已有 Cocotb/Verilator 输出与 `docs/exps/cam_retrieval_noise_sweep_*.md`。 +硬件检索周期检测命令： + +```bash +just cam-test-retrieval-artifact outputs/cam_retrieval_benchmark/datasets/cifar10_hash512_rows512_queries128.npz 512 +``` + ## 2. 软件检索速度与质量 软件路径只计时 Top-K 匹配阶段，不包含 `.npz` 加载、指标聚合和结果写盘。 @@ -119,7 +125,51 @@ just remote "python scripts/sw_retrieval_benchmark.py --dataset outputs/cam_retr - 50% 噪声后检索基本不可用。 - 与 CIFAR-10 相同，Golden Match@5 在低噪声下也迅速接近 0，说明精确排序比类别命中率更脆弱。 -## 5. 指标解释 +## 5. 硬件检索周期性能 + +本次已将硬件周期检测合并进真实 `.npz` 检索数据集的 Cocotb/Verilator +benchmark。实现位置： + +- `hw/sim/tests/top/utils.py::query_topk_once_with_latency` +- `hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py::cam_retrieval_benchmark` + +周期口径如下： + +| 指标 | 含义 | +|---|---| +| `accept_to_first_result_cycles` | query 被 `query_valid && query_ready` 接受后，到首个 `result_valid` beat 的周期数。 | +| `accept_to_last_result_cycles` | query 被接受后，到 `result_last` 断言的周期数，即完整 Top-K 串行结果输出完成。 | +| `total_query_cycles` | 从拉高 `query_valid` 到 Top-K 输出完成的完整事务周期数。 | +| `cycles_per_query` | 当前报告中的主指标，等于平均 `accept_to_last_result_cycles`。 | +| `queries_per_cycle` | `num_queries / total_query_cycles`，用于观察仿真事务吞吐率。 | + +选择 `accept_to_last_result_cycles` 作为主 `cycles_per_query` 的原因是：Top-K +检索只有在串行结果流输出到 `result_last` 后才算完整完成；仅用首个 +`result_valid` 会低估实际 Top-K 检索事务成本。 + +### CIFAR-10 无噪声硬件周期结果 + +配置：512 rows × 128 queries × 512-bit hash，`TOPK_K=5`，`LANES=8`， +`WRITE_NOISE_EN=0`。 + +| 数据集 | 模式 | 查询数 | accept→first | accept→last / cycles/query | total query cycles | queries/cycle | 状态 | +|---|---|---:|---:|---:|---:|---:|---| +| CIFAR-10 | hardware no-noise | 128 | 1027.000000 | 1031.000000 | 132096 | 0.000968992 | pass | + +对应日志标记： + +```text +RETRIEVAL_PERF_RESULT mode=no_noise num_queries=128 cycles_per_query=1031.000000 accept_to_first_result_cycles=1027.000000 accept_to_last_result_cycles=1031.000000 total_query_cycles=132096 queries_per_cycle=0.000968992 status=pass +``` + +该结果说明：在当前 `NUM_ROWS=512, LANES=8, TOPK_K=5` 的硬件仿真配置下， +一次完整 Top-K 检索事务约为 **1031 cycles/query**。首个结果 beat 约在 +1027 cycles 后出现，完整 Top-K 输出额外消耗约 4 个周期。 + +> 注：以上数据来自 Verilator/Cocotb 仿真，不是 FPGA 板上实测。它可用于 +> 架构级周期趋势分析，但不能直接等同于板级频率、吞吐或端到端系统延迟。 + +## 6. 指标解释 | 指标 | 含义 | |---|---| @@ -130,8 +180,10 @@ just remote "python scripts/sw_retrieval_benchmark.py --dataset outputs/cam_retr | Golden Match@K / `exact_match_rate` | Top-K row index 列表是否与参考模型完全一致。该指标比 Hit@K 更严格。 | | `ns_per_query` | 软件 Top-K 匹配阶段平均耗时；不含加载和写盘。 | | `queries_per_second` | 软件 Top-K 匹配阶段吞吐率。 | +| `cycles_per_query` | 硬件仿真中一次完整 Top-K 检索事务的平均周期数，当前采用 accept→last 口径。 | +| `queries_per_cycle` | 硬件仿真中完成 query 数除以总 query 事务周期数。 | -## 6. 当前结论 +## 7. 当前结论 1. **软件汉明距离基线已可作为硬件 CAM 检索的功能参考。** 在无噪声条件下，硬件仿真和软件基线的质量指标及 Golden Match 均一致。 @@ -139,8 +191,9 @@ just remote "python scripts/sw_retrieval_benchmark.py --dataset outputs/cam_retr 2. **当前软件基线速度约为 4.86k queries/s。** 该结果来自 Python integer brute-force Hamming scan，数据规模为 512 rows × 128 queries × 512 bits。 -3. **硬件检索质量基准目前主要报告质量指标，不报告 cycles/query。** - 现有硬件性能基准中已有 cycle 统计逻辑，但尚未和真实 retrieval dataset 的 Top-K benchmark 合并。因此本文不报告硬件 query/s 或 cycles/query。 +3. **硬件检索质量基准现在已经报告 cycles/query。** + 在 CIFAR-10 512-row/128-query 无噪声配置下，完整 Top-K 检索为 + **1031 cycles/query**，首个结果为 **1027 cycles/query**。 4. **写噪声对精确排序影响显著。** 即使 Hit@K 保持较高，Golden Match@K 也会快速下降，说明噪声首先破坏精确排序，再进一步破坏类别命中。 @@ -148,13 +201,13 @@ just remote "python scripts/sw_retrieval_benchmark.py --dataset outputs/cam_retr 5. **CIFAR-100 比 CIFAR-10 更能体现检索难度。** 在无噪声下 CIFAR-100 的 Hit@1/Hit@5 分别为 0.695312/0.867188，明显低于 CIFAR-10 的 1.0/1.0，更适合作为后续检索质量对比主数据集。 -## 7. 后续建议 +## 8. 后续建议 -1. 将 `hw/sim/tests/perf/test_cam_perf.py::query_once_with_latency` 的周期测量逻辑合并到 retrieval benchmark，记录真实数据集上的 `cycles/query`。 +1. 对 CIFAR-100 也运行同样的硬件周期检测，补齐与软件质量表同尺度的硬件周期表。 2. 在 `docs/exps` 中继续维护： - 软件检索速度表； - 硬件无噪声一致性表； - 硬件噪声鲁棒性表； - - 后续硬件 `cycles/query` 表。 + - 硬件 `cycles/query` 表。 3. 对软件基线补充 NumPy/PyTorch vectorized Hamming scan，以区分“朴素 Python baseline”和“优化软件 baseline”。 4. 增加 `NUM_ROWS` sweep：例如 512、1024、2048、4096 rows，观察软件 brute-force scan 的线性增长趋势。 diff --git a/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py b/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py index 2e1b275..40bd454 100644 --- a/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py +++ b/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py @@ -20,7 +20,7 @@ from tests.top.utils import ( dut_lanes, dut_num_rows, get_param, - query_topk_once, + query_topk_once_with_latency, reset_dut, write_rows, ) @@ -46,6 +46,13 @@ class RetrievalDataset: seed: int +@dataclass(frozen=True) +class QueryTiming: + accept_to_first_result_cycles: int + accept_to_last_result_cycles: int + total_query_cycles: int + + @dataclass(frozen=True) class MetricAccumulator: precision_sum: float = 0.0 @@ -203,6 +210,38 @@ def mode_from_params(write_noise_en: int) -> str: return "no_noise" +def summarize_query_timings(timings: list[QueryTiming]) -> dict[str, float]: + if not timings: + return { + "num_queries": 0, + "total_query_cycles": 0, + "mean_total_query_cycles": 0.0, + "min_total_query_cycles": 0, + "max_total_query_cycles": 0, + "mean_accept_to_first_result_cycles": 0.0, + "mean_accept_to_last_result_cycles": 0.0, + "cycles_per_query": 0.0, + "queries_per_cycle": 0.0, + } + + total_cycles = sum(t.total_query_cycles for t in timings) + total_first = sum(t.accept_to_first_result_cycles for t in timings) + total_last = sum(t.accept_to_last_result_cycles for t in timings) + count = len(timings) + mean_last = total_last / float(count) + return { + "num_queries": count, + "total_query_cycles": total_cycles, + "mean_total_query_cycles": total_cycles / float(count), + "min_total_query_cycles": min(t.total_query_cycles for t in timings), + "max_total_query_cycles": max(t.total_query_cycles for t in timings), + "mean_accept_to_first_result_cycles": total_first / float(count), + "mean_accept_to_last_result_cycles": mean_last, + "cycles_per_query": mean_last, + "queries_per_cycle": count / float(total_cycles) if total_cycles > 0 else 0.0, + } + + def output_dir_for(mode: str) -> Path: run_id = os.environ.get("CAM_RETRIEVAL_RUN_ID") if not run_id: @@ -225,7 +264,9 @@ def write_outputs(out_dir: Path, result: dict) -> None: "write_noise_en", "write_noise_rate_num", "write_noise_rate_den", "num_queries", "k", "macro_precision", "retrieval_recall", "macro_f1", - "recall@k", "exact_match_rate", "status", + "recall@k", "exact_match_rate", "cycles_per_query", + "mean_accept_to_first_result_cycles", "mean_accept_to_last_result_cycles", + "mean_total_query_cycles", "total_query_cycles", "queries_per_cycle", "status", ] with metrics_csv.open("w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) @@ -248,6 +289,16 @@ def write_outputs(out_dir: Path, result: dict) -> None: "macro_f1": metrics["macro_f1"], "recall@k": metrics["recall@k"], "exact_match_rate": metrics["exact_match_rate"], + "cycles_per_query": result.get("performance", {}).get("cycles_per_query", ""), + "mean_accept_to_first_result_cycles": result.get("performance", {}).get( + "mean_accept_to_first_result_cycles", "", + ), + "mean_accept_to_last_result_cycles": result.get("performance", {}).get( + "mean_accept_to_last_result_cycles", "", + ), + "mean_total_query_cycles": result.get("performance", {}).get("mean_total_query_cycles", ""), + "total_query_cycles": result.get("performance", {}).get("total_query_cycles", ""), + "queries_per_cycle": result.get("performance", {}).get("queries_per_cycle", ""), "status": result["status"], } writer.writerow(row) @@ -260,6 +311,16 @@ def write_outputs(out_dir: Path, result: dict) -> None: f"- status: `{result['status']}`", f"- num_queries: `{result['dataset']['num_queries']}`", "", + "## Hardware performance", + "", + f"- cycles_per_query: `{result.get('performance', {}).get('cycles_per_query', '')}`", + f"- accept_to_first_result_cycles: `{result.get('performance', {}).get('mean_accept_to_first_result_cycles', '')}`", + f"- accept_to_last_result_cycles: `{result.get('performance', {}).get('mean_accept_to_last_result_cycles', '')}`", + f"- total_query_cycles: `{result.get('performance', {}).get('total_query_cycles', '')}`", + f"- queries_per_cycle: `{result.get('performance', {}).get('queries_per_cycle', '')}`", + "", + "## Retrieval quality", + "", "| k | macro_precision | retrieval_recall | macro_f1 | recall@k | exact_match_rate |", "|---:|---:|---:|---:|---:|---:|", ] @@ -300,9 +361,11 @@ async def cam_retrieval_benchmark(dut): await write_rows(dut, dataset.rows) accumulators = {k: MetricAccumulator() for k in BENCHMARK_KS} + timings: list[QueryTiming] = [] for query, query_label in zip(dataset.queries, dataset.query_labels): - beats, _, _, _ = await query_topk_once(dut, query) + beats, _, _, _, timing = await query_topk_once_with_latency(dut, query) + timings.append(QueryTiming(**timing)) if len(beats) < max(BENCHMARK_KS): raise AssertionError(f"Expected at least {max(BENCHMARK_KS)} Top-K beats, got {len(beats)}") @@ -339,6 +402,7 @@ async def cam_retrieval_benchmark(dut): "seed": dataset.seed, }, "metrics": {str(k): accumulators[k].as_dict() for k in BENCHMARK_KS}, + "performance": summarize_query_timings(timings), } out_dir = output_dir_for(mode) @@ -353,6 +417,21 @@ async def cam_retrieval_benchmark(dut): str(out_dir.relative_to(_project_root())), ) + performance = result["performance"] + dut._log.info( + "RETRIEVAL_PERF_RESULT mode=%s num_queries=%d cycles_per_query=%.6f " + "accept_to_first_result_cycles=%.6f accept_to_last_result_cycles=%.6f " + "total_query_cycles=%d queries_per_cycle=%.9f status=pass output_dir=%s", + mode, + performance["num_queries"], + performance["cycles_per_query"], + performance["mean_accept_to_first_result_cycles"], + performance["mean_accept_to_last_result_cycles"], + performance["total_query_cycles"], + performance["queries_per_cycle"], + str(out_dir.relative_to(_project_root())), + ) + if write_noise_en == 0: assert result["metrics"]["5"]["exact_match_rate"] == 1.0, ( f"Expected perfect exact match with no noise, got " diff --git a/hw/sim/tests/top/utils.py b/hw/sim/tests/top/utils.py index e31adb6..26ce642 100644 --- a/hw/sim/tests/top/utils.py +++ b/hw/sim/tests/top/utils.py @@ -15,7 +15,7 @@ CAM 顶层测试的共享辅助函数。 from __future__ import annotations import numpy as np -from cocotb.triggers import RisingEdge +from cocotb.triggers import ReadOnly, RisingEdge, Timer from model.ref_model import ( # noqa: E402 match_top1, unpack_score_debug_flat, @@ -202,28 +202,73 @@ async def query_topk_once(dut, query, timeout_cycles=None): - beats: [(rank, row, score, last), ...] - score_debug: np.ndarray 或 None（SIM_DEBUG 模式） """ + beats, top1_index, top1_score, score_debug, _ = await query_topk_once_with_latency( + dut, query, timeout_cycles=timeout_cycles, + ) + return beats, top1_index, top1_score, score_debug + + +async def query_topk_once_with_latency(dut, query, timeout_cycles=None): + """发起一次查询、收集完整 Top-K 结果流，并返回周期计数。 + + 返回：(beats, top1_index, top1_score, score_debug, timing) + + ``timing`` 字段： + - accept_to_first_result_cycles: query 接受到首个 result_valid beat + - accept_to_last_result_cycles: query 接受到 result_last beat（Top-K 完成） + - total_query_cycles: 从拉高 query_valid 到 Top-K 完成的总上升沿数 + + ``query_ready`` 是组合信号，握手周期在上升沿前采样；结果信号在 + ReadOnly settled phase 采样，避免重新引入 query_ready 采样时序问题。 + """ await wait_idle(dut) dut.query_hash.value = int(query) # 等待 query_ready 为高（DUT 已就绪），避免组合逻辑下降沿导致的 - # valid&&ready 握手丢失问题 + # valid&&ready 握手丢失问题。 while not int(dut.query_ready.value): await RisingEdge(dut.clk) - # assert query_valid 覆盖一个上升沿完成握手 + edge_count = 0 dut.query_valid.value = 1 + dut.result_ready.value = 1 + await RisingEdge(dut.clk) + edge_count += 1 + q_ready = int(dut.query_ready.value) + assert q_ready, "Query accept handshake was missed despite query_ready pre-wait" + accept_edge = edge_count dut.query_valid.value = 0 - # 若调用者未指定超时，根据 DUT 参数动态估算 if timeout_cycles is None: timeout_cycles = dut_query_timeout_cycles(dut) - # 消费完整串行结果流 - beats = await collect_topk(dut, timeout_cycles=timeout_cycles) + beats = [] + first_result_edge = None + last_result_edge = None + + for _ in range(timeout_cycles): + await RisingEdge(dut.clk) + edge_count += 1 + await ReadOnly() + if int(dut.result_valid.value): + if first_result_edge is None: + first_result_edge = edge_count + rank = int(dut.result_rank.value) + row = int(dut.result_row.value) + score = int(dut.result_score.value) + last = int(dut.result_last.value) + beats.append((rank, row, score, last)) + if last: + last_result_edge = edge_count + await Timer(1, units="step") + break + await Timer(1, units="step") + + if first_result_edge is None or last_result_edge is None: + raise AssertionError("Top-K result stream did not finish") - # score_debug 在查询完成后可用（需 SIM_DEBUG 编译） num_rows = dut_num_rows(dut) score_bits = dut_score_bits(dut) score_debug = None @@ -234,7 +279,12 @@ async def query_topk_once(dut, query, timeout_cycles=None): score_bits, ) - return beats, beats[0][1], beats[0][2], score_debug + timing = { + "accept_to_first_result_cycles": int(first_result_edge - accept_edge), + "accept_to_last_result_cycles": int(last_result_edge - accept_edge), + "total_query_cycles": int(edge_count), + } + return beats, beats[0][1], beats[0][2], score_debug, timing async def query_once(dut, query, timeout_cycles=None): diff --git a/tests/test_retrieval_benchmark_performance.py b/tests/test_retrieval_benchmark_performance.py new file mode 100644 index 0000000..82272f3 --- /dev/null +++ b/tests/test_retrieval_benchmark_performance.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import csv +import sys +from pathlib import Path + + +HW_SIM_DIR = Path(__file__).resolve().parents[1] / "hw" / "sim" +if str(HW_SIM_DIR) not in sys.path: + sys.path.insert(0, str(HW_SIM_DIR)) + +from benchmarks.retrieval.test_retrieval_benchmark import ( # noqa: E402 + QueryTiming, + summarize_query_timings, + write_outputs, +) + + +def test_summarize_query_timings_reports_topk_completion_headline() -> None: + summary = summarize_query_timings([ + QueryTiming( + accept_to_first_result_cycles=10, + accept_to_last_result_cycles=14, + total_query_cycles=16, + ), + QueryTiming( + accept_to_first_result_cycles=12, + accept_to_last_result_cycles=18, + total_query_cycles=21, + ), + ]) + + assert summary == { + "num_queries": 2, + "total_query_cycles": 37, + "mean_total_query_cycles": 18.5, + "min_total_query_cycles": 16, + "max_total_query_cycles": 21, + "mean_accept_to_first_result_cycles": 11.0, + "mean_accept_to_last_result_cycles": 16.0, + "cycles_per_query": 16.0, + "queries_per_cycle": 2 / 37, + } + + +def test_write_outputs_includes_hardware_performance_fields(tmp_path: Path) -> None: + result = { + "run_id": "test-run", + "mode": "no_noise", + "status": "pass", + "params": { + "num_rows": 512, + "hash_bits": 512, + "lanes": 8, + "topk_k": 5, + "write_noise_en": 0, + "write_noise_rate_num": 0, + "write_noise_rate_den": 100, + }, + "dataset": { + "num_classes": 10, + "positives_per_class": 0, + "queries_per_class": 0, + "num_queries": 2, + "seed": 0, + }, + "metrics": { + "1": { + "macro_precision": 1.0, + "retrieval_recall": 0.5, + "macro_f1": 2 / 3, + "recall@k": 1.0, + "exact_match_rate": 1.0, + } + }, + "performance": { + "num_queries": 2, + "total_query_cycles": 37, + "mean_total_query_cycles": 18.5, + "min_total_query_cycles": 16, + "max_total_query_cycles": 21, + "mean_accept_to_first_result_cycles": 11.0, + "mean_accept_to_last_result_cycles": 16.0, + "cycles_per_query": 16.0, + "queries_per_cycle": 2 / 37, + }, + } + + write_outputs(tmp_path, result) + + with (tmp_path / "metrics.csv").open(newline="", encoding="utf-8") as f: + row = next(csv.DictReader(f)) + + assert row["cycles_per_query"] == "16.0" + assert row["mean_accept_to_first_result_cycles"] == "11.0" + assert row["mean_accept_to_last_result_cycles"] == "16.0" + assert row["queries_per_cycle"] == str(2 / 37) + + summary = (tmp_path / "summary.md").read_text(encoding="utf-8") + assert "## Hardware performance" in summary + assert "cycles_per_query" in summary + assert "accept_to_last_result_cycles" in summary