SikongJueluo/Mini-Nav
1
0
Fork 0
mirror of https://github.com/SikongJueluo/Mini-Nav.git synced 2026-07-12 20:15:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(retrieval): add CAM retrieval benchmark with topk scoring and read noise support

Browse Source 
- Add cocotb benchmark infrastructure under hw/sim/benchmarks/retrieval/ with Makefile
- Implement test_retrieval_benchmark.py supporting configurable topk-k, read/write noise
- Add cluster-based synthetic dataset generator with configurable bit-flip rates
- Add reference model functions: match_topk, match_topk_from_scores, score_rows_with_read_noise
- Add .justfile shortcuts: cam-test-retrieval-no-noise, cam-test-retrieval-read-noise
- Add TOPK_K to Verilator EXTRA_ARGS via cocotb-common.mk
- Add unit tests for topk sorting logic and stateful read-noise scoring
This commit is contained in:
SikongJueluo
2026-05-22 19:01:43 +08:00
parent 29f4cc91f6
commit e1bed00cc4
8 changed files with 503 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.justfile
View File

@@ -73,3 +73,11 @@ cam-test-module MODULE:
# Run a single test case within a module (e.g., cam-test MODULE=cam_core_banked TESTCASE=banked_core_reads_aligned_eight_row_batch_after_one_cycle)
cam-test MODULE TESTCASE:
just remote "make -C hw/sim clean && make -C hw/sim test-module MODULE={{ MODULE }} COCOTB_TESTCASE={{ TESTCASE }}"
# Run CAM retrieval benchmark without hardware noise
cam-test-retrieval-no-noise:
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 WRITE_NOISE_EN=0 READ_NOISE_EN=0"
# Run CAM retrieval benchmark with read noise enabled
cam-test-retrieval-read-noise:
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 WRITE_NOISE_EN=0 READ_NOISE_EN=1 READ_NOISE_RATE_NUM=1 READ_NOISE_RATE_DEN=100 READ_NOISE_BITS=8"

7
hw/sim/Makefile
View File

@@ -2,7 +2,7 @@ PYTHON ?= python
MODULE_TESTS := cam_core_banked candidate_fifo match_engine_pipeline cam_write_noise cam_read_noise popcount_pipeline topk_tracker result_serializer
TOP_CONFIGS := no_noise write_noise read_noise
.PHONY: help test-all test-top test-top-all test-modules test-module test-model test-perf clean $(MODULE_TESTS:%=test-module-%) $(TOP_CONFIGS:%=test-top-%)
.PHONY: help test-all test-top test-top-all test-modules test-module test-model test-perf clean $(MODULE_TESTS:%=test-module-%) $(TOP_CONFIGS:%=test-top-%) test-benchmark-retrieval
help:
@echo "Available hw/sim targets:"
@@ -15,6 +15,7 @@ help:
@echo " make test-module MODULE=cam_core_banked"
@echo " make test-modules"
@echo " make test-perf"
@echo " make test-benchmark-retrieval # 检索质量 benchmark（非默认）"
@echo " make test-all"
@echo " make clean"
@@ -42,6 +43,9 @@ test-model:
test-perf:
$(MAKE) -C tests/perf
test-benchmark-retrieval:
$(MAKE) -C benchmarks/retrieval
clean:
@for config in $(TOP_CONFIGS); do \
$(MAKE) -C tests/top/$$config clean || exit $$?; \
@@ -49,5 +53,6 @@ clean:
@for module in $(MODULE_TESTS); do \
$(MAKE) -C tests/modules/$$module clean || exit $$?; \
done
$(MAKE) -C benchmarks/retrieval clean
$(MAKE) -C tests/perf clean
rm -rf .pytest_cache tests/model/.pytest_cache

13
hw/sim/benchmarks/retrieval/Makefile Normal file
View File

@@ -0,0 +1,13 @@
SIM_ROOT := $(abspath ../..)
RTL_ROOT := $(abspath $(SIM_ROOT)/../rtl)
include $(SIM_ROOT)/mk/rtl-sources.mk
TOPLEVEL := cam_top
COCOTB_TEST_MODULES := benchmarks.retrieval.test_retrieval_benchmark
VERILOG_SOURCES := $(RTL_CAM_TOP)
TOPK_K ?= 5
WRITE_NOISE_EN ?= 0
READ_NOISE_EN ?= 0
include $(SIM_ROOT)/mk/cocotb-common.mk

0
hw/sim/benchmarks/retrieval/__init__.py Normal file
View File

340
hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py Normal file
View File

@@ -0,0 +1,340 @@
from __future__ import annotations
import csv
import json
import os
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
import cocotb
import numpy as np
from cocotb.clock import Clock
from model.ref_model import (
lane_seed_128,
match_topk,
match_topk_from_scores,
score_rows_with_read_noise,
)
from tests.top.utils import (
dut_hash_bits,
dut_lanes,
dut_num_rows,
get_param,
query_topk_once,
reset_dut,
write_rows,
)
MAX_BENCHMARK_QUERIES = 128
DEFAULT_POSITIVES_PER_CLASS = 8
DEFAULT_QUERIES_PER_CLASS = 2
DEFAULT_ROW_FLIP_BITS = 16
DEFAULT_QUERY_FLIP_BITS = 16
DEFAULT_SEED = 20260522
BENCHMARK_KS = (1, 5)
@dataclass(frozen=True)
class RetrievalDataset:
rows: list[int]
row_labels: list[int]
queries: list[int]
query_labels: list[int]
num_classes: int
positives_per_class: int
queries_per_class: int
seed: int
@dataclass(frozen=True)
class MetricAccumulator:
precision_sum: float = 0.0
recall_sum: float = 0.0
f1_sum: float = 0.0
exact_matches: int = 0
count: int = 0
def add(self, precision: float, recall: float, f1: float, exact: bool) -> "MetricAccumulator":
return MetricAccumulator(
precision_sum=self.precision_sum + precision,
recall_sum=self.recall_sum + recall,
f1_sum=self.f1_sum + f1,
exact_matches=self.exact_matches + int(exact),
count=self.count + 1,
)
def as_dict(self) -> dict[str, float]:
if self.count == 0:
return {
"macro_precision": 0.0,
"macro_recall": 0.0,
"macro_f1": 0.0,
"exact_match_rate": 0.0,
}
return {
"macro_precision": self.precision_sum / self.count,
"macro_recall": self.recall_sum / self.count,
"macro_f1": self.f1_sum / self.count,
"exact_match_rate": self.exact_matches / self.count,
}
def _project_root() -> Path:
return Path(__file__).resolve().parents[4]
def _flip_exact_bits(rng: np.random.Generator, width: int, n_bits: int) -> int:
n_bits = max(0, min(int(n_bits), int(width)))
if n_bits == 0:
return 0
positions = rng.choice(width, size=n_bits, replace=False)
mask = 0
for pos in positions:
mask |= 1 << int(pos)
return mask
def make_clustered_dataset(
*,
num_rows: int,
hash_bits: int,
positives_per_class: int = DEFAULT_POSITIVES_PER_CLASS,
queries_per_class: int = DEFAULT_QUERIES_PER_CLASS,
row_flip_bits: int = DEFAULT_ROW_FLIP_BITS,
query_flip_bits: int = DEFAULT_QUERY_FLIP_BITS,
seed: int = DEFAULT_SEED,
) -> RetrievalDataset:
usable_rows = int(num_rows)
if usable_rows < 5:
raise AssertionError("Retrieval benchmark requires at least 5 CAM rows")
positives_per_class = min(positives_per_class, usable_rows)
num_classes = max(1, usable_rows // positives_per_class)
usable_rows = num_classes * positives_per_class
# Cap total queries to keep simulation runtime bounded
max_queries = min(MAX_BENCHMARK_QUERIES, num_classes * queries_per_class)
if max_queries < num_classes * queries_per_class:
queries_per_class = max(1, max_queries // num_classes)
rng = np.random.default_rng(seed)
mask = (1 << hash_bits) - 1
words = (hash_bits + 63) // 64
rows: list[int] = []
row_labels: list[int] = []
queries: list[int] = []
query_labels: list[int] = []
for class_id in range(num_classes):
center = 0
for word in range(words):
center |= int(rng.integers(0, 1 << 64, dtype=np.uint64)) << (64 * word)
center &= mask
for _ in range(positives_per_class):
rows.append((center ^ _flip_exact_bits(rng, hash_bits, row_flip_bits)) & mask)
row_labels.append(class_id)
for _ in range(queries_per_class):
queries.append((center ^ _flip_exact_bits(rng, hash_bits, query_flip_bits)) & mask)
query_labels.append(class_id)
return RetrievalDataset(
rows=rows,
row_labels=row_labels,
queries=queries,
query_labels=query_labels,
num_classes=num_classes,
positives_per_class=positives_per_class,
queries_per_class=queries_per_class,
seed=seed,
)
def compute_metrics(topk_indices: list[int], row_labels: list[int], query_label: int, k: int) -> tuple[float, float, float]:
retrieved = topk_indices[:k]
relevant = {idx for idx, label in enumerate(row_labels) if label == query_label}
tp = len(set(retrieved) & relevant)
precision = tp / float(k)
recall = tp / float(len(relevant)) if relevant else 0.0
f1 = 0.0 if precision + recall == 0 else (2.0 * precision * recall) / (precision + recall)
return precision, recall, f1
def mode_from_params(write_noise_en: int, read_noise_en: int) -> str:
if write_noise_en and read_noise_en:
return "write_read_noise"
if write_noise_en:
return "write_noise"
if read_noise_en:
return "read_noise"
return "no_noise"
def output_dir_for(mode: str) -> Path:
run_id = os.environ.get("CAM_RETRIEVAL_RUN_ID")
if not run_id:
run_id = f"{datetime.now().strftime('%Y-%m-%d-%H%M%S')}-{mode}"
out_dir = _project_root() / "outputs" / "cam_retrieval_benchmark" / run_id
out_dir.mkdir(parents=True, exist_ok=True)
(out_dir / "logs").mkdir(exist_ok=True)
return out_dir
def write_outputs(out_dir: Path, result: dict) -> None:
metrics_json = out_dir / "metrics.json"
metrics_csv = out_dir / "metrics.csv"
summary_md = out_dir / "summary.md"
metrics_json.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
fieldnames = [
"run_id", "mode", "num_rows", "hash_bits", "lanes", "topk_k",
"write_noise_en", "read_noise_en", "write_noise_rate_num",
"write_noise_rate_den", "read_noise_rate_num", "read_noise_rate_den",
"num_queries", "k", "macro_precision", "macro_recall", "macro_f1",
"exact_match_rate", "status",
]
with metrics_csv.open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
for k, metrics in result["metrics"].items():
row = {
"run_id": result["run_id"],
"mode": result["mode"],
"num_rows": result["params"]["num_rows"],
"hash_bits": result["params"]["hash_bits"],
"lanes": result["params"]["lanes"],
"topk_k": result["params"]["topk_k"],
"write_noise_en": result["params"]["write_noise_en"],
"read_noise_en": result["params"]["read_noise_en"],
"write_noise_rate_num": result["params"]["write_noise_rate_num"],
"write_noise_rate_den": result["params"]["write_noise_rate_den"],
"read_noise_rate_num": result["params"]["read_noise_rate_num"],
"read_noise_rate_den": result["params"]["read_noise_rate_den"],
"num_queries": result["dataset"]["num_queries"],
"k": int(k),
"macro_precision": metrics["macro_precision"],
"macro_recall": metrics["macro_recall"],
"macro_f1": metrics["macro_f1"],
"exact_match_rate": metrics["exact_match_rate"],
"status": result["status"],
}
writer.writerow(row)
lines = [
"# CAM Retrieval Benchmark Summary",
"",
f"- run_id: `{result['run_id']}`",
f"- mode: `{result['mode']}`",
f"- status: `{result['status']}`",
f"- num_queries: `{result['dataset']['num_queries']}`",
"",
"| k | macro_precision | macro_recall | macro_f1 | exact_match_rate |",
"|---:|---:|---:|---:|---:|",
]
for k, metrics in result["metrics"].items():
lines.append(
f"| {k} | {metrics['macro_precision']:.6f} | {metrics['macro_recall']:.6f} | "
f"{metrics['macro_f1']:.6f} | {metrics['exact_match_rate']:.6f} |"
)
lines.extend([
"",
"说明：结果来自 Verilator/Cocotb 仿真，不是 FPGA 板上实测。",
])
summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
@cocotb.test()
async def cam_retrieval_benchmark(dut):
cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start())
await reset_dut(dut)
num_rows = dut_num_rows(dut)
hash_bits = dut_hash_bits(dut)
lanes = dut_lanes(dut)
write_noise_en = int(get_param(dut, "WRITE_NOISE_EN", 0) or 0)
read_noise_en = int(get_param(dut, "READ_NOISE_EN", 0) or 0)
write_noise_rate_num = int(get_param(dut, "WRITE_NOISE_RATE_NUM", 0) or 0)
write_noise_rate_den = int(get_param(dut, "WRITE_NOISE_RATE_DEN", 100) or 100)
read_noise_rate_num = int(get_param(dut, "READ_NOISE_RATE_NUM", 0) or 0)
read_noise_rate_den = int(get_param(dut, "READ_NOISE_RATE_DEN", 100) or 100)
read_noise_bits = int(get_param(dut, "READ_NOISE_BITS", 8) or 8)
mode = mode_from_params(write_noise_en, read_noise_en)
if write_noise_en:
raise AssertionError("First retrieval benchmark version only supports WRITE_NOISE_EN=0")
if num_rows % lanes != 0:
raise AssertionError("Retrieval benchmark requires NUM_ROWS divisible by LANES")
dataset = make_clustered_dataset(num_rows=num_rows, hash_bits=hash_bits)
await write_rows(dut, dataset.rows)
accumulators = {k: MetricAccumulator() for k in BENCHMARK_KS}
read_lane_states = [lane_seed_128(0x6A09_E667_F3BC_C909, lane) for lane in range(lanes)]
for query, query_label in zip(dataset.queries, dataset.query_labels):
beats, _, _, _ = await query_topk_once(dut, query)
if len(beats) < max(BENCHMARK_KS):
raise AssertionError(f"Expected at least {max(BENCHMARK_KS)} Top-K beats, got {len(beats)}")
dut_topk = [int(beat[1]) for beat in beats[: max(BENCHMARK_KS)]]
if read_noise_en:
scores, read_lane_states = score_rows_with_read_noise(
query, dataset.rows, lane_states=read_lane_states,
width=hash_bits, lanes=lanes, noise_bits=read_noise_bits,
rate_num=read_noise_rate_num, rate_den=read_noise_rate_den,
)
golden_topk = match_topk_from_scores(scores, max(BENCHMARK_KS))
else:
golden_topk, _ = match_topk(query, dataset.rows, width=hash_bits, k=max(BENCHMARK_KS))
for k in BENCHMARK_KS:
precision, recall, f1 = compute_metrics(dut_topk, dataset.row_labels, query_label, k)
exact = dut_topk[:k] == golden_topk[:k]
accumulators[k] = accumulators[k].add(precision, recall, f1, exact)
run_id = os.environ.get("CAM_RETRIEVAL_RUN_ID") or f"{datetime.now().strftime('%Y-%m-%d-%H%M%S')}-{mode}"
result = {
"run_id": run_id,
"mode": mode,
"status": "pass",
"params": {
"num_rows": len(dataset.rows),
"hash_bits": hash_bits,
"lanes": lanes,
"topk_k": max(BENCHMARK_KS),
"write_noise_en": write_noise_en,
"read_noise_en": read_noise_en,
"write_noise_rate_num": write_noise_rate_num,
"write_noise_rate_den": write_noise_rate_den,
"read_noise_rate_num": read_noise_rate_num,
"read_noise_rate_den": read_noise_rate_den,
},
"dataset": {
"num_classes": dataset.num_classes,
"positives_per_class": dataset.positives_per_class,
"queries_per_class": dataset.queries_per_class,
"num_queries": len(dataset.queries),
"seed": dataset.seed,
},
"metrics": {str(k): accumulators[k].as_dict() for k in BENCHMARK_KS},
}
out_dir = output_dir_for(mode)
write_outputs(out_dir, result)
for k in BENCHMARK_KS:
metrics = result["metrics"][str(k)]
dut._log.info(
"RETRIEVAL_RESULT mode=%s k=%d precision=%.6f recall=%.6f f1=%.6f exact_match=%.6f output_dir=%s",
mode, k, metrics["macro_precision"], metrics["macro_recall"],
metrics["macro_f1"], metrics["exact_match_rate"],
str(out_dir.relative_to(_project_root())),
)
assert result["metrics"]["5"]["exact_match_rate"] == 1.0

3
hw/sim/mk/cocotb-common.mk
View File

@@ -24,8 +24,9 @@ TOPLEVEL_LANG ?= verilog
NUM_ROWS ?= 4096
HASH_BITS ?= 512
LANES ?= 8
TOPK_K ?= 4
EXTRA_ARGS += +define+NUM_ROWS=$(NUM_ROWS) +define+HASH_BITS=$(HASH_BITS) +define+LANES=$(LANES)
EXTRA_ARGS += +define+NUM_ROWS=$(NUM_ROWS) +define+HASH_BITS=$(HASH_BITS) +define+LANES=$(LANES) +define+TOPK_K=$(TOPK_K)
EXTRA_ARGS += --trace --trace-fst --trace-structs
COMPILE_ARGS += -Wall -Wno-fatal

67
hw/sim/model/ref_model.py
View File

@@ -53,6 +53,30 @@ def match_top1(
)
def match_topk_from_scores(scores: Sequence[int], k: int) -> list[int]:
"""Return row indices sorted by score desc, row index asc (HW tie-break)."""
if k <= 0:
raise ValueError("k must be greater than 0")
return sorted(range(len(scores)), key=lambda idx: (-int(scores[idx]), idx))[: min(k, len(scores))]
def match_topk(
query: int,
rows: Sequence[int],
*,
width: int = 512,
k: int = 5,
) -> tuple[list[int], np.ndarray]:
"""Pure Top-K matching — noise is already baked into rows if needed.
Returns (list of row indices in rank order, NumPy score array).
"""
scores = np.zeros(len(rows), dtype=np.int32)
for idx, row in enumerate(rows):
scores[idx] = xnor_popcount_score(int(query), int(row), width)
return match_topk_from_scores(scores, k), scores
def xorshift128(state: int) -> int:
"""128-bit xorshift PRNG, single step. Matches random128.sv."""
mask32 = (1 << 32) - 1
@@ -182,6 +206,49 @@ def generate_read_lane_masks(
return masks, next_states
def score_rows_with_read_noise(
query: int,
rows: Sequence[int],
*,
lane_states: Sequence[int],
width: int = 512,
lanes: int = 8,
noise_bits: int = 8,
rate_num: int = 1,
rate_den: int = 100,
) -> tuple[np.ndarray, list[int]]:
"""Score one query with read noise and return updated lane PRNG states.
Unlike match_top1_with_read_noise(), this helper is stateful across calls:
callers pass current lane states in and receive the next states back.
This matches a DUT that is reset once, then serves multiple queries.
"""
assert lanes > 0
assert len(rows) % lanes == 0
assert len(lane_states) == lanes
scores = np.zeros(len(rows), dtype=np.int32)
next_lane_states = [int(state) for state in lane_states]
for base in range(0, len(rows), lanes):
lane_valid = [True] * lanes
masks, next_lane_states = generate_read_lane_masks(
next_lane_states,
hash_bits=width,
noise_bits=noise_bits,
rate_num=rate_num,
rate_den=rate_den,
lane_valid=lane_valid,
)
for lane in range(lanes):
row_idx = base + lane
noisy_row = int(rows[row_idx]) ^ int(masks[lane])
scores[row_idx] = xnor_popcount_score(int(query), noisy_row, width)
return scores, next_lane_states
def match_top1_with_read_noise(
query: int,
rows: Sequence[int],

67
hw/sim/tests/model/test_ref_model_noise.py
View File

@@ -152,3 +152,70 @@ def test_read_noise_model_is_reproducible_after_reset_seed():
assert first.top1_index == second.top1_index
assert first.top1_score == second.top1_score
assert first.scores.tolist() == second.scores.tolist()
# ==============================================================================
# 测试 5：Top-K 排序 — 分数降序、平局行号升序
# ==============================================================================
def test_match_topk_from_scores_uses_score_desc_then_row_asc():
"""Top-K 排序规则：分数越大越优先；分数相同时行号越小越优先。"""
from model.ref_model import match_topk_from_scores
import numpy as np
scores = np.array([7, 9, 9, 2, 7], dtype=np.int32)
assert match_topk_from_scores(scores, 4) == [1, 2, 0, 4]
def test_match_topk_scores_rows_by_xnor_popcount():
"""match_topk 通过 xnor_popcount 计算分数，返回排序后的行索引和分数数组。"""
from model.ref_model import match_topk
rows = [0b0000, 0b1111, 0b0011, 0b0101]
query = 0b0000
indices, scores = match_topk(query, rows, width=4, k=3)
assert scores.tolist() == [4, 0, 2, 2]
assert indices == [0, 2, 3]
def test_match_topk_clamps_k_to_row_count():
"""当 k 超过实际行数时，返回所有行（按排序）。"""
from model.ref_model import match_topk
indices, scores = match_topk(0, [0, 1], width=1, k=5)
assert scores.tolist() == [1, 0]
assert indices == [0, 1]
# ==============================================================================
# 测试 6：读取噪声 stateful 评分助手的跨查询状态推进
# ==============================================================================
def test_score_rows_with_read_noise_stateful_across_queries():
"""score_rows_with_read_noise 在多次调用间正确推进 lane PRNG 状态。
两次调用使用相同的 rows/query 和零噪声率：
- 分数应一致（无噪声翻转）
- 但 lane states 应该变化（PRNG 已推进）
"""
from model.ref_model import score_rows_with_read_noise
rows = [0, 0, 0, 0]
query = 0
lane_states = [1, 2]
scores_1, next_states_1 = score_rows_with_read_noise(
query, rows, lane_states=lane_states, width=128, lanes=2,
noise_bits=2, rate_num=0, rate_den=100,
)
scores_2, next_states_2 = score_rows_with_read_noise(
query, rows, lane_states=next_states_1, width=128, lanes=2,
noise_bits=2, rate_num=0, rate_den=100,
)
assert scores_1.tolist() == [128, 128, 128, 128]
assert scores_2.tolist() == [128, 128, 128, 128]
assert next_states_1 != lane_states
assert next_states_2 != next_states_1