diff --git a/.justfile b/.justfile index 726ddcc..6bba46d 100644 --- a/.justfile +++ b/.justfile @@ -73,3 +73,11 @@ cam-test-module MODULE: # Run a single test case within a module (e.g., cam-test MODULE=cam_core_banked TESTCASE=banked_core_reads_aligned_eight_row_batch_after_one_cycle) cam-test MODULE TESTCASE: just remote "make -C hw/sim clean && make -C hw/sim test-module MODULE={{ MODULE }} COCOTB_TESTCASE={{ TESTCASE }}" + +# Run CAM retrieval benchmark without hardware noise +cam-test-retrieval-no-noise: + just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 WRITE_NOISE_EN=0 READ_NOISE_EN=0" + +# Run CAM retrieval benchmark with read noise enabled +cam-test-retrieval-read-noise: + just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 WRITE_NOISE_EN=0 READ_NOISE_EN=1 READ_NOISE_RATE_NUM=1 READ_NOISE_RATE_DEN=100 READ_NOISE_BITS=8" diff --git a/hw/sim/Makefile b/hw/sim/Makefile index d054d40..107fa52 100644 --- a/hw/sim/Makefile +++ b/hw/sim/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python MODULE_TESTS := cam_core_banked candidate_fifo match_engine_pipeline cam_write_noise cam_read_noise popcount_pipeline topk_tracker result_serializer TOP_CONFIGS := no_noise write_noise read_noise -.PHONY: help test-all test-top test-top-all test-modules test-module test-model test-perf clean $(MODULE_TESTS:%=test-module-%) $(TOP_CONFIGS:%=test-top-%) +.PHONY: help test-all test-top test-top-all test-modules test-module test-model test-perf clean $(MODULE_TESTS:%=test-module-%) $(TOP_CONFIGS:%=test-top-%) test-benchmark-retrieval help: @echo "Available hw/sim targets:" @@ -15,6 +15,7 @@ help: @echo " make test-module MODULE=cam_core_banked" @echo " make test-modules" @echo " make test-perf" + @echo " make test-benchmark-retrieval # 检索质量 benchmark（非默认）" @echo " make test-all" @echo " make clean" @@ -42,6 +43,9 @@ test-model: test-perf: $(MAKE) -C tests/perf +test-benchmark-retrieval: + $(MAKE) -C benchmarks/retrieval + clean: @for config in $(TOP_CONFIGS); do \ $(MAKE) -C tests/top/$$config clean || exit $$?; \ @@ -49,5 +53,6 @@ clean: @for module in $(MODULE_TESTS); do \ $(MAKE) -C tests/modules/$$module clean || exit $$?; \ done + $(MAKE) -C benchmarks/retrieval clean $(MAKE) -C tests/perf clean rm -rf .pytest_cache tests/model/.pytest_cache diff --git a/hw/sim/benchmarks/retrieval/Makefile b/hw/sim/benchmarks/retrieval/Makefile new file mode 100644 index 0000000..c300be7 --- /dev/null +++ b/hw/sim/benchmarks/retrieval/Makefile @@ -0,0 +1,13 @@ +SIM_ROOT := $(abspath ../..) +RTL_ROOT := $(abspath $(SIM_ROOT)/../rtl) +include $(SIM_ROOT)/mk/rtl-sources.mk + +TOPLEVEL := cam_top +COCOTB_TEST_MODULES := benchmarks.retrieval.test_retrieval_benchmark +VERILOG_SOURCES := $(RTL_CAM_TOP) + +TOPK_K ?= 5 +WRITE_NOISE_EN ?= 0 +READ_NOISE_EN ?= 0 + +include $(SIM_ROOT)/mk/cocotb-common.mk diff --git a/hw/sim/benchmarks/retrieval/__init__.py b/hw/sim/benchmarks/retrieval/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py b/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py new file mode 100644 index 0000000..f2db9ac --- /dev/null +++ b/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py @@ -0,0 +1,340 @@ +from __future__ import annotations + +import csv +import json +import os +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +import cocotb +import numpy as np +from cocotb.clock import Clock + +from model.ref_model import ( + lane_seed_128, + match_topk, + match_topk_from_scores, + score_rows_with_read_noise, +) +from tests.top.utils import ( + dut_hash_bits, + dut_lanes, + dut_num_rows, + get_param, + query_topk_once, + reset_dut, + write_rows, +) + +MAX_BENCHMARK_QUERIES = 128 +DEFAULT_POSITIVES_PER_CLASS = 8 +DEFAULT_QUERIES_PER_CLASS = 2 +DEFAULT_ROW_FLIP_BITS = 16 +DEFAULT_QUERY_FLIP_BITS = 16 +DEFAULT_SEED = 20260522 +BENCHMARK_KS = (1, 5) + + +@dataclass(frozen=True) +class RetrievalDataset: + rows: list[int] + row_labels: list[int] + queries: list[int] + query_labels: list[int] + num_classes: int + positives_per_class: int + queries_per_class: int + seed: int + + +@dataclass(frozen=True) +class MetricAccumulator: + precision_sum: float = 0.0 + recall_sum: float = 0.0 + f1_sum: float = 0.0 + exact_matches: int = 0 + count: int = 0 + + def add(self, precision: float, recall: float, f1: float, exact: bool) -> "MetricAccumulator": + return MetricAccumulator( + precision_sum=self.precision_sum + precision, + recall_sum=self.recall_sum + recall, + f1_sum=self.f1_sum + f1, + exact_matches=self.exact_matches + int(exact), + count=self.count + 1, + ) + + def as_dict(self) -> dict[str, float]: + if self.count == 0: + return { + "macro_precision": 0.0, + "macro_recall": 0.0, + "macro_f1": 0.0, + "exact_match_rate": 0.0, + } + return { + "macro_precision": self.precision_sum / self.count, + "macro_recall": self.recall_sum / self.count, + "macro_f1": self.f1_sum / self.count, + "exact_match_rate": self.exact_matches / self.count, + } + + +def _project_root() -> Path: + return Path(__file__).resolve().parents[4] + + +def _flip_exact_bits(rng: np.random.Generator, width: int, n_bits: int) -> int: + n_bits = max(0, min(int(n_bits), int(width))) + if n_bits == 0: + return 0 + positions = rng.choice(width, size=n_bits, replace=False) + mask = 0 + for pos in positions: + mask |= 1 << int(pos) + return mask + + +def make_clustered_dataset( + *, + num_rows: int, + hash_bits: int, + positives_per_class: int = DEFAULT_POSITIVES_PER_CLASS, + queries_per_class: int = DEFAULT_QUERIES_PER_CLASS, + row_flip_bits: int = DEFAULT_ROW_FLIP_BITS, + query_flip_bits: int = DEFAULT_QUERY_FLIP_BITS, + seed: int = DEFAULT_SEED, +) -> RetrievalDataset: + usable_rows = int(num_rows) + if usable_rows < 5: + raise AssertionError("Retrieval benchmark requires at least 5 CAM rows") + + positives_per_class = min(positives_per_class, usable_rows) + num_classes = max(1, usable_rows // positives_per_class) + usable_rows = num_classes * positives_per_class + + # Cap total queries to keep simulation runtime bounded + max_queries = min(MAX_BENCHMARK_QUERIES, num_classes * queries_per_class) + if max_queries < num_classes * queries_per_class: + queries_per_class = max(1, max_queries // num_classes) + + rng = np.random.default_rng(seed) + mask = (1 << hash_bits) - 1 + words = (hash_bits + 63) // 64 + + rows: list[int] = [] + row_labels: list[int] = [] + queries: list[int] = [] + query_labels: list[int] = [] + + for class_id in range(num_classes): + center = 0 + for word in range(words): + center |= int(rng.integers(0, 1 << 64, dtype=np.uint64)) << (64 * word) + center &= mask + + for _ in range(positives_per_class): + rows.append((center ^ _flip_exact_bits(rng, hash_bits, row_flip_bits)) & mask) + row_labels.append(class_id) + + for _ in range(queries_per_class): + queries.append((center ^ _flip_exact_bits(rng, hash_bits, query_flip_bits)) & mask) + query_labels.append(class_id) + + return RetrievalDataset( + rows=rows, + row_labels=row_labels, + queries=queries, + query_labels=query_labels, + num_classes=num_classes, + positives_per_class=positives_per_class, + queries_per_class=queries_per_class, + seed=seed, + ) + + +def compute_metrics(topk_indices: list[int], row_labels: list[int], query_label: int, k: int) -> tuple[float, float, float]: + retrieved = topk_indices[:k] + relevant = {idx for idx, label in enumerate(row_labels) if label == query_label} + tp = len(set(retrieved) & relevant) + precision = tp / float(k) + recall = tp / float(len(relevant)) if relevant else 0.0 + f1 = 0.0 if precision + recall == 0 else (2.0 * precision * recall) / (precision + recall) + return precision, recall, f1 + + +def mode_from_params(write_noise_en: int, read_noise_en: int) -> str: + if write_noise_en and read_noise_en: + return "write_read_noise" + if write_noise_en: + return "write_noise" + if read_noise_en: + return "read_noise" + return "no_noise" + + +def output_dir_for(mode: str) -> Path: + run_id = os.environ.get("CAM_RETRIEVAL_RUN_ID") + if not run_id: + run_id = f"{datetime.now().strftime('%Y-%m-%d-%H%M%S')}-{mode}" + out_dir = _project_root() / "outputs" / "cam_retrieval_benchmark" / run_id + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "logs").mkdir(exist_ok=True) + return out_dir + + +def write_outputs(out_dir: Path, result: dict) -> None: + metrics_json = out_dir / "metrics.json" + metrics_csv = out_dir / "metrics.csv" + summary_md = out_dir / "summary.md" + + metrics_json.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + fieldnames = [ + "run_id", "mode", "num_rows", "hash_bits", "lanes", "topk_k", + "write_noise_en", "read_noise_en", "write_noise_rate_num", + "write_noise_rate_den", "read_noise_rate_num", "read_noise_rate_den", + "num_queries", "k", "macro_precision", "macro_recall", "macro_f1", + "exact_match_rate", "status", + ] + with metrics_csv.open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for k, metrics in result["metrics"].items(): + row = { + "run_id": result["run_id"], + "mode": result["mode"], + "num_rows": result["params"]["num_rows"], + "hash_bits": result["params"]["hash_bits"], + "lanes": result["params"]["lanes"], + "topk_k": result["params"]["topk_k"], + "write_noise_en": result["params"]["write_noise_en"], + "read_noise_en": result["params"]["read_noise_en"], + "write_noise_rate_num": result["params"]["write_noise_rate_num"], + "write_noise_rate_den": result["params"]["write_noise_rate_den"], + "read_noise_rate_num": result["params"]["read_noise_rate_num"], + "read_noise_rate_den": result["params"]["read_noise_rate_den"], + "num_queries": result["dataset"]["num_queries"], + "k": int(k), + "macro_precision": metrics["macro_precision"], + "macro_recall": metrics["macro_recall"], + "macro_f1": metrics["macro_f1"], + "exact_match_rate": metrics["exact_match_rate"], + "status": result["status"], + } + writer.writerow(row) + + lines = [ + "# CAM Retrieval Benchmark Summary", + "", + f"- run_id: `{result['run_id']}`", + f"- mode: `{result['mode']}`", + f"- status: `{result['status']}`", + f"- num_queries: `{result['dataset']['num_queries']}`", + "", + "| k | macro_precision | macro_recall | macro_f1 | exact_match_rate |", + "|---:|---:|---:|---:|---:|", + ] + for k, metrics in result["metrics"].items(): + lines.append( + f"| {k} | {metrics['macro_precision']:.6f} | {metrics['macro_recall']:.6f} | " + f"{metrics['macro_f1']:.6f} | {metrics['exact_match_rate']:.6f} |" + ) + lines.extend([ + "", + "说明：结果来自 Verilator/Cocotb 仿真，不是 FPGA 板上实测。", + ]) + summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +@cocotb.test() +async def cam_retrieval_benchmark(dut): + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_dut(dut) + + num_rows = dut_num_rows(dut) + hash_bits = dut_hash_bits(dut) + lanes = dut_lanes(dut) + write_noise_en = int(get_param(dut, "WRITE_NOISE_EN", 0) or 0) + read_noise_en = int(get_param(dut, "READ_NOISE_EN", 0) or 0) + write_noise_rate_num = int(get_param(dut, "WRITE_NOISE_RATE_NUM", 0) or 0) + write_noise_rate_den = int(get_param(dut, "WRITE_NOISE_RATE_DEN", 100) or 100) + read_noise_rate_num = int(get_param(dut, "READ_NOISE_RATE_NUM", 0) or 0) + read_noise_rate_den = int(get_param(dut, "READ_NOISE_RATE_DEN", 100) or 100) + read_noise_bits = int(get_param(dut, "READ_NOISE_BITS", 8) or 8) + mode = mode_from_params(write_noise_en, read_noise_en) + + if write_noise_en: + raise AssertionError("First retrieval benchmark version only supports WRITE_NOISE_EN=0") + if num_rows % lanes != 0: + raise AssertionError("Retrieval benchmark requires NUM_ROWS divisible by LANES") + + dataset = make_clustered_dataset(num_rows=num_rows, hash_bits=hash_bits) + await write_rows(dut, dataset.rows) + + accumulators = {k: MetricAccumulator() for k in BENCHMARK_KS} + read_lane_states = [lane_seed_128(0x6A09_E667_F3BC_C909, lane) for lane in range(lanes)] + + for query, query_label in zip(dataset.queries, dataset.query_labels): + beats, _, _, _ = await query_topk_once(dut, query) + if len(beats) < max(BENCHMARK_KS): + raise AssertionError(f"Expected at least {max(BENCHMARK_KS)} Top-K beats, got {len(beats)}") + + dut_topk = [int(beat[1]) for beat in beats[: max(BENCHMARK_KS)]] + + if read_noise_en: + scores, read_lane_states = score_rows_with_read_noise( + query, dataset.rows, lane_states=read_lane_states, + width=hash_bits, lanes=lanes, noise_bits=read_noise_bits, + rate_num=read_noise_rate_num, rate_den=read_noise_rate_den, + ) + golden_topk = match_topk_from_scores(scores, max(BENCHMARK_KS)) + else: + golden_topk, _ = match_topk(query, dataset.rows, width=hash_bits, k=max(BENCHMARK_KS)) + + for k in BENCHMARK_KS: + precision, recall, f1 = compute_metrics(dut_topk, dataset.row_labels, query_label, k) + exact = dut_topk[:k] == golden_topk[:k] + accumulators[k] = accumulators[k].add(precision, recall, f1, exact) + + run_id = os.environ.get("CAM_RETRIEVAL_RUN_ID") or f"{datetime.now().strftime('%Y-%m-%d-%H%M%S')}-{mode}" + result = { + "run_id": run_id, + "mode": mode, + "status": "pass", + "params": { + "num_rows": len(dataset.rows), + "hash_bits": hash_bits, + "lanes": lanes, + "topk_k": max(BENCHMARK_KS), + "write_noise_en": write_noise_en, + "read_noise_en": read_noise_en, + "write_noise_rate_num": write_noise_rate_num, + "write_noise_rate_den": write_noise_rate_den, + "read_noise_rate_num": read_noise_rate_num, + "read_noise_rate_den": read_noise_rate_den, + }, + "dataset": { + "num_classes": dataset.num_classes, + "positives_per_class": dataset.positives_per_class, + "queries_per_class": dataset.queries_per_class, + "num_queries": len(dataset.queries), + "seed": dataset.seed, + }, + "metrics": {str(k): accumulators[k].as_dict() for k in BENCHMARK_KS}, + } + + out_dir = output_dir_for(mode) + write_outputs(out_dir, result) + + for k in BENCHMARK_KS: + metrics = result["metrics"][str(k)] + dut._log.info( + "RETRIEVAL_RESULT mode=%s k=%d precision=%.6f recall=%.6f f1=%.6f exact_match=%.6f output_dir=%s", + mode, k, metrics["macro_precision"], metrics["macro_recall"], + metrics["macro_f1"], metrics["exact_match_rate"], + str(out_dir.relative_to(_project_root())), + ) + + assert result["metrics"]["5"]["exact_match_rate"] == 1.0 diff --git a/hw/sim/mk/cocotb-common.mk b/hw/sim/mk/cocotb-common.mk index 8cc7d97..dec44d7 100644 --- a/hw/sim/mk/cocotb-common.mk +++ b/hw/sim/mk/cocotb-common.mk @@ -24,8 +24,9 @@ TOPLEVEL_LANG ?= verilog NUM_ROWS ?= 4096 HASH_BITS ?= 512 LANES ?= 8 +TOPK_K ?= 4 -EXTRA_ARGS += +define+NUM_ROWS=$(NUM_ROWS) +define+HASH_BITS=$(HASH_BITS) +define+LANES=$(LANES) +EXTRA_ARGS += +define+NUM_ROWS=$(NUM_ROWS) +define+HASH_BITS=$(HASH_BITS) +define+LANES=$(LANES) +define+TOPK_K=$(TOPK_K) EXTRA_ARGS += --trace --trace-fst --trace-structs COMPILE_ARGS += -Wall -Wno-fatal diff --git a/hw/sim/model/ref_model.py b/hw/sim/model/ref_model.py index 1a3a01c..4ca6659 100644 --- a/hw/sim/model/ref_model.py +++ b/hw/sim/model/ref_model.py @@ -53,6 +53,30 @@ def match_top1( ) +def match_topk_from_scores(scores: Sequence[int], k: int) -> list[int]: + """Return row indices sorted by score desc, row index asc (HW tie-break).""" + if k <= 0: + raise ValueError("k must be greater than 0") + return sorted(range(len(scores)), key=lambda idx: (-int(scores[idx]), idx))[: min(k, len(scores))] + + +def match_topk( + query: int, + rows: Sequence[int], + *, + width: int = 512, + k: int = 5, +) -> tuple[list[int], np.ndarray]: + """Pure Top-K matching — noise is already baked into rows if needed. + + Returns (list of row indices in rank order, NumPy score array). + """ + scores = np.zeros(len(rows), dtype=np.int32) + for idx, row in enumerate(rows): + scores[idx] = xnor_popcount_score(int(query), int(row), width) + return match_topk_from_scores(scores, k), scores + + def xorshift128(state: int) -> int: """128-bit xorshift PRNG, single step. Matches random128.sv.""" mask32 = (1 << 32) - 1 @@ -182,6 +206,49 @@ def generate_read_lane_masks( return masks, next_states +def score_rows_with_read_noise( + query: int, + rows: Sequence[int], + *, + lane_states: Sequence[int], + width: int = 512, + lanes: int = 8, + noise_bits: int = 8, + rate_num: int = 1, + rate_den: int = 100, +) -> tuple[np.ndarray, list[int]]: + """Score one query with read noise and return updated lane PRNG states. + + Unlike match_top1_with_read_noise(), this helper is stateful across calls: + callers pass current lane states in and receive the next states back. + This matches a DUT that is reset once, then serves multiple queries. + """ + assert lanes > 0 + assert len(rows) % lanes == 0 + assert len(lane_states) == lanes + + scores = np.zeros(len(rows), dtype=np.int32) + next_lane_states = [int(state) for state in lane_states] + + for base in range(0, len(rows), lanes): + lane_valid = [True] * lanes + masks, next_lane_states = generate_read_lane_masks( + next_lane_states, + hash_bits=width, + noise_bits=noise_bits, + rate_num=rate_num, + rate_den=rate_den, + lane_valid=lane_valid, + ) + + for lane in range(lanes): + row_idx = base + lane + noisy_row = int(rows[row_idx]) ^ int(masks[lane]) + scores[row_idx] = xnor_popcount_score(int(query), noisy_row, width) + + return scores, next_lane_states + + def match_top1_with_read_noise( query: int, rows: Sequence[int], diff --git a/hw/sim/tests/model/test_ref_model_noise.py b/hw/sim/tests/model/test_ref_model_noise.py index d824eaf..1afb087 100644 --- a/hw/sim/tests/model/test_ref_model_noise.py +++ b/hw/sim/tests/model/test_ref_model_noise.py @@ -152,3 +152,70 @@ def test_read_noise_model_is_reproducible_after_reset_seed(): assert first.top1_index == second.top1_index assert first.top1_score == second.top1_score assert first.scores.tolist() == second.scores.tolist() + + +# ============================================================================== +# 测试 5：Top-K 排序 — 分数降序、平局行号升序 +# ============================================================================== + + +def test_match_topk_from_scores_uses_score_desc_then_row_asc(): + """Top-K 排序规则：分数越大越优先；分数相同时行号越小越优先。""" + from model.ref_model import match_topk_from_scores + import numpy as np + + scores = np.array([7, 9, 9, 2, 7], dtype=np.int32) + assert match_topk_from_scores(scores, 4) == [1, 2, 0, 4] + + +def test_match_topk_scores_rows_by_xnor_popcount(): + """match_topk 通过 xnor_popcount 计算分数，返回排序后的行索引和分数数组。""" + from model.ref_model import match_topk + + rows = [0b0000, 0b1111, 0b0011, 0b0101] + query = 0b0000 + indices, scores = match_topk(query, rows, width=4, k=3) + assert scores.tolist() == [4, 0, 2, 2] + assert indices == [0, 2, 3] + + +def test_match_topk_clamps_k_to_row_count(): + """当 k 超过实际行数时，返回所有行（按排序）。""" + from model.ref_model import match_topk + + indices, scores = match_topk(0, [0, 1], width=1, k=5) + assert scores.tolist() == [1, 0] + assert indices == [0, 1] + + +# ============================================================================== +# 测试 6：读取噪声 stateful 评分助手的跨查询状态推进 +# ============================================================================== + + +def test_score_rows_with_read_noise_stateful_across_queries(): + """score_rows_with_read_noise 在多次调用间正确推进 lane PRNG 状态。 + + 两次调用使用相同的 rows/query 和零噪声率： + - 分数应一致（无噪声翻转） + - 但 lane states 应该变化（PRNG 已推进） + """ + from model.ref_model import score_rows_with_read_noise + + rows = [0, 0, 0, 0] + query = 0 + lane_states = [1, 2] + + scores_1, next_states_1 = score_rows_with_read_noise( + query, rows, lane_states=lane_states, width=128, lanes=2, + noise_bits=2, rate_num=0, rate_den=100, + ) + scores_2, next_states_2 = score_rows_with_read_noise( + query, rows, lane_states=next_states_1, width=128, lanes=2, + noise_bits=2, rate_num=0, rate_den=100, + ) + + assert scores_1.tolist() == [128, 128, 128, 128] + assert scores_2.tolist() == [128, 128, 128, 128] + assert next_states_1 != lane_states + assert next_states_2 != next_states_1