diff --git a/.gitignore b/.gitignore index 0cff4fc..48590e9 100644 --- a/.gitignore +++ b/.gitignore @@ -221,6 +221,7 @@ openspec/changes/ .codegraph/ .logs/ docs/superpowers +.workspace # Devenv .devenv* diff --git a/.justfile b/.justfile index b3dbcd1..6f8e0be 100644 --- a/.justfile +++ b/.justfile @@ -97,3 +97,30 @@ cam-test-retrieval-artifact DATASET_PATH NUM_ROWS="4096": # Run CAM retrieval benchmark on a prepared artifact with write noise enabled (Phase 2: read noise removed) cam-test-retrieval-artifact-write-noise DATASET_PATH NUM_ROWS="4096": just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 NUM_ROWS={{NUM_ROWS}} WRITE_NOISE_EN=1 WRITE_NOISE_RATE_NUM=1 WRITE_NOISE_RATE_DEN=100 CAM_RETRIEVAL_DATASET={{ DATASET_PATH }}" + +# ── CAM retrieval benchmark noise sweep ──────────────────────────────────────── + +# Run noise sweep on a prepared dataset (0%–100%, step 10%) +# Usage: just cam-benchmark-retrieval-sweep DATASET=outputs/.../cifar10_hash512_rows512_queries128.npz NUM_ROWS=512 +cam-benchmark-retrieval-sweep DATASET NUM_ROWS="512": + just remote "python scripts/run_retrieval_noise_sweep.py --dataset {{DATASET}} --num-rows {{NUM_ROWS}} --output docs/cam_retrieval_noise_sweep.md" + +# Prepare CIFAR10 dataset + run full noise sweep (all-in-one) +cam-benchmark-sweep-cifar10 ROWS="512" QUERIES="128": + just cam-prepare-retrieval-cifar10 {{ROWS}} {{QUERIES}} + just remote "python scripts/run_retrieval_noise_sweep.py --dataset outputs/cam_retrieval_benchmark/datasets/cifar10_hash512_rows{{ROWS}}_queries{{QUERIES}}.npz --num-rows {{ROWS}} --output docs/cam_retrieval_noise_sweep_cifar10.md" + +# Prepare CIFAR100 dataset + run full noise sweep (all-in-one) +cam-benchmark-sweep-cifar100 ROWS="512" QUERIES="128": + just cam-prepare-retrieval-cifar100 {{ROWS}} {{QUERIES}} + just remote "python scripts/run_retrieval_noise_sweep.py --dataset outputs/cam_retrieval_benchmark/datasets/cifar100_hash512_rows{{ROWS}}_queries{{QUERIES}}.npz --num-rows {{ROWS}} --output docs/cam_retrieval_noise_sweep_cifar100.md" + +# ── Remote ↔ local sync ──────────────────────────────────────────────────────── + +# Download benchmark outputs from remote +download-outputs: + rsync {{ rsync_flags }} {{ remote_root }}/outputs/cam_retrieval_benchmark/ outputs/cam_retrieval_benchmark/ + +# Download docs from remote +download-docs: + rsync {{ rsync_flags }} {{ remote_root }}/docs/ docs/ diff --git a/docs/exps/cam_retrieval_noise_sweep_cifar10.md b/docs/exps/cam_retrieval_noise_sweep_cifar10.md new file mode 100644 index 0000000..a9aba09 --- /dev/null +++ b/docs/exps/cam_retrieval_noise_sweep_cifar10.md @@ -0,0 +1,95 @@ +# CAM Retrieval Benchmark — Noise Sweep Summary + +**Generated:** 2026-05-27 19:00:42 + +## Configuration + +| Parameter | Value | +|---|---| +| Dataset path | `outputs/cam_retrieval_benchmark/datasets/cifar10_hash512_rows512_queries128.npz` | +| NUM_ROWS | 512 | +| TOPK_K | 5 | +| HASH_BITS | 512 | +| Noise rates | 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100% | +| Total runs | 11 | +| Passed | 11 | +| Failed | 0 | + +### Dataset Details + +| Field | Value | +|---|---| +| num_queries | 128 | +| num_classes | 10 | +| seed | 0 | + +--- + +## Results by Noise Rate + +### k=1 + +| Noise (%) | WRITE_NOISE_EN | NUM/DEN | Hit@K | Precision@K | Hit-F1@K | Std-Recall@K | Std-F1@K | Golden Match@K | Status | +|---|---:|---|---:|---|---:|---|---:|---|---:|---| +| 0% | 0 | — | 1.000000 | 1.000000 | 1.000000 | 0.019531 | 0.038314 | 1.000000 | ✓ | +| 10% | 1 | 10/100 | 1.000000 | 1.000000 | 1.000000 | 0.019531 | 0.038314 | 0.507812 | ✓ | +| 20% | 1 | 20/100 | 1.000000 | 1.000000 | 1.000000 | 0.019531 | 0.038314 | 0.234375 | ✓ | +| 30% | 1 | 30/100 | 0.992188 | 0.992188 | 0.992188 | 0.019378 | 0.038014 | 0.164062 | ✓ | +| 40% | 1 | 40/100 | 0.984375 | 0.984375 | 0.984375 | 0.019228 | 0.037719 | 0.093750 | ✓ | +| 50% | 1 | 50/100 | 0.257812 | 0.257812 | 0.257812 | 0.005043 | 0.009893 | 0.023438 | ✓ | +| 60% | 1 | 60/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 70% | 1 | 70/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 80% | 1 | 80/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 90% | 1 | 90/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 100% | 1 | 100/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | + +### k=5 + +| Noise (%) | WRITE_NOISE_EN | NUM/DEN | Hit@K | Precision@K | Hit-F1@K | Std-Recall@K | Std-F1@K | Golden Match@K | Status | +|---|---:|---|---:|---|---:|---|---:|---|---:|---| +| 0% | 0 | — | 1.000000 | 1.000000 | 1.000000 | 0.097656 | 0.177936 | 1.000000 | ✓ | +| 10% | 1 | 10/100 | 1.000000 | 1.000000 | 1.000000 | 0.097656 | 0.177936 | 0.000000 | ✓ | +| 20% | 1 | 20/100 | 1.000000 | 1.000000 | 1.000000 | 0.097656 | 0.177936 | 0.000000 | ✓ | +| 30% | 1 | 30/100 | 1.000000 | 0.995313 | 0.997651 | 0.097197 | 0.177099 | 0.000000 | ✓ | +| 40% | 1 | 40/100 | 1.000000 | 0.939062 | 0.968574 | 0.091729 | 0.167132 | 0.000000 | ✓ | +| 50% | 1 | 50/100 | 0.750000 | 0.234375 | 0.357143 | 0.022913 | 0.041745 | 0.000000 | ✓ | +| 60% | 1 | 60/100 | 0.015625 | 0.003125 | 0.005208 | 0.000306 | 0.000558 | 0.000000 | ✓ | +| 70% | 1 | 70/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 80% | 1 | 80/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 90% | 1 | 90/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 100% | 1 | 100/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | + +--- + +## Cross-Noise Comparison (primary: Hit@K) + +| Noise (%) | Hit@1 | Hit@5 | Δ(Hit@1 vs 0%) | Δ(Hit@5 vs 0%) | +|---|---:|---:|---:|---:| +| 0% | 1.000000 | 1.000000 | +0.000000 | +0.000000 | +| 10% | 1.000000 | 1.000000 | +0.000000 | +0.000000 | +| 20% | 1.000000 | 1.000000 | +0.000000 | +0.000000 | +| 30% | 0.992188 | 1.000000 | -0.007812 | +0.000000 | +| 40% | 0.984375 | 1.000000 | -0.015625 | +0.000000 | +| 50% | 0.257812 | 0.750000 | -0.742188 | -0.250000 | +| 60% | 0.000000 | 0.015625 | -1.000000 | -0.984375 | +| 70% | 0.000000 | 0.000000 | -1.000000 | -1.000000 | +| 80% | 0.000000 | 0.000000 | -1.000000 | -1.000000 | +| 90% | 0.000000 | 0.000000 | -1.000000 | -1.000000 | +| 100% | 0.000000 | 0.000000 | -1.000000 | -1.000000 | + +--- + +## Metric Definitions + +- **Hit@K**: fraction of queries where at least one relevant item appears in Top-K results (primary metric). +- **Precision@K**: mean per-query precision — averaged `tp/k` across all queries. +- **Hit-F1@K**: `2 × Hit@K × Precision@K / (Hit@K + Precision@K)` — F1 using hit-rate recall. +- **Std-Recall@K**: mean per-query standard retrieval recall — `tp / |relevant|` averaged across queries (supplementary). +- **Std-F1@K**: `2 × Precision@K × Std-Recall@K / (Precision@K + Std-Recall@K)` — F1 using standard recall (supplementary). +- **Golden Match@K**: fraction of queries where DUT Top-K exactly matches the reference golden Top-K. + +The paper uses Hit@K and Precision@K as primary metrics. Std-Recall@K and Std-F1@K are supplementary, +included to show Top-K coverage against all relevant items in the database. + +*Results from Verilator/Cocotb simulation. Not measured on physical FPGA hardware.* + diff --git a/docs/exps/cam_retrieval_noise_sweep_cifar100.md b/docs/exps/cam_retrieval_noise_sweep_cifar100.md new file mode 100644 index 0000000..ce6967c --- /dev/null +++ b/docs/exps/cam_retrieval_noise_sweep_cifar100.md @@ -0,0 +1,95 @@ +# CAM Retrieval Benchmark — Noise Sweep Summary + +**Generated:** 2026-05-27 19:00:46 + +## Configuration + +| Parameter | Value | +|---|---| +| Dataset path | `outputs/cam_retrieval_benchmark/datasets/cifar100_hash512_rows512_queries128.npz` | +| NUM_ROWS | 512 | +| TOPK_K | 5 | +| HASH_BITS | 512 | +| Noise rates | 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100% | +| Total runs | 11 | +| Passed | 11 | +| Failed | 0 | + +### Dataset Details + +| Field | Value | +|---|---| +| num_queries | 128 | +| num_classes | 100 | +| seed | 0 | + +--- + +## Results by Noise Rate + +### k=1 + +| Noise (%) | WRITE_NOISE_EN | NUM/DEN | Hit@K | Precision@K | Hit-F1@K | Std-Recall@K | Std-F1@K | Golden Match@K | Status | +|---|---:|---|---:|---|---:|---|---:|---|---:|---| +| 0% | 0 | — | 0.695312 | 0.695312 | 0.695312 | 0.134635 | 0.225589 | 1.000000 | ✓ | +| 10% | 1 | 10/100 | 0.585938 | 0.585938 | 0.585938 | 0.113281 | 0.189857 | 0.593750 | ✓ | +| 20% | 1 | 20/100 | 0.562500 | 0.562500 | 0.562500 | 0.109115 | 0.182774 | 0.460938 | ✓ | +| 30% | 1 | 30/100 | 0.460938 | 0.460938 | 0.460938 | 0.088802 | 0.148915 | 0.304688 | ✓ | +| 40% | 1 | 40/100 | 0.234375 | 0.234375 | 0.234375 | 0.044792 | 0.075210 | 0.101562 | ✓ | +| 50% | 1 | 50/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 60% | 1 | 60/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 70% | 1 | 70/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 80% | 1 | 80/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 90% | 1 | 90/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 100% | 1 | 100/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | + +### k=5 + +| Noise (%) | WRITE_NOISE_EN | NUM/DEN | Hit@K | Precision@K | Hit-F1@K | Std-Recall@K | Std-F1@K | Golden Match@K | Status | +|---|---:|---|---:|---|---:|---|---:|---|---:|---| +| 0% | 0 | — | 0.867188 | 0.462500 | 0.603261 | 0.445052 | 0.453608 | 1.000000 | ✓ | +| 10% | 1 | 10/100 | 0.812500 | 0.421875 | 0.555380 | 0.405990 | 0.413780 | 0.023438 | ✓ | +| 20% | 1 | 20/100 | 0.742188 | 0.364062 | 0.488502 | 0.350000 | 0.356893 | 0.000000 | ✓ | +| 30% | 1 | 30/100 | 0.640625 | 0.248437 | 0.358029 | 0.238802 | 0.243525 | 0.000000 | ✓ | +| 40% | 1 | 40/100 | 0.460938 | 0.117187 | 0.186867 | 0.113021 | 0.115066 | 0.000000 | ✓ | +| 50% | 1 | 50/100 | 0.062500 | 0.015625 | 0.025000 | 0.014844 | 0.015224 | 0.000000 | ✓ | +| 60% | 1 | 60/100 | 0.007812 | 0.001563 | 0.002604 | 0.001563 | 0.001563 | 0.000000 | ✓ | +| 70% | 1 | 70/100 | 0.007812 | 0.001563 | 0.002604 | 0.001563 | 0.001563 | 0.000000 | ✓ | +| 80% | 1 | 80/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 90% | 1 | 90/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | +| 100% | 1 | 100/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ | + +--- + +## Cross-Noise Comparison (primary: Hit@K) + +| Noise (%) | Hit@1 | Hit@5 | Δ(Hit@1 vs 0%) | Δ(Hit@5 vs 0%) | +|---|---:|---:|---:|---:| +| 0% | 0.695312 | 0.867188 | +0.000000 | +0.000000 | +| 10% | 0.585938 | 0.812500 | -0.109375 | -0.054688 | +| 20% | 0.562500 | 0.742188 | -0.132812 | -0.125000 | +| 30% | 0.460938 | 0.640625 | -0.234375 | -0.226562 | +| 40% | 0.234375 | 0.460938 | -0.460938 | -0.406250 | +| 50% | 0.000000 | 0.062500 | -0.695312 | -0.804688 | +| 60% | 0.000000 | 0.007812 | -0.695312 | -0.859375 | +| 70% | 0.000000 | 0.007812 | -0.695312 | -0.859375 | +| 80% | 0.000000 | 0.000000 | -0.695312 | -0.867188 | +| 90% | 0.000000 | 0.000000 | -0.695312 | -0.867188 | +| 100% | 0.000000 | 0.000000 | -0.695312 | -0.867188 | + +--- + +## Metric Definitions + +- **Hit@K**: fraction of queries where at least one relevant item appears in Top-K results (primary metric). +- **Precision@K**: mean per-query precision — averaged `tp/k` across all queries. +- **Hit-F1@K**: `2 × Hit@K × Precision@K / (Hit@K + Precision@K)` — F1 using hit-rate recall. +- **Std-Recall@K**: mean per-query standard retrieval recall — `tp / |relevant|` averaged across queries (supplementary). +- **Std-F1@K**: `2 × Precision@K × Std-Recall@K / (Precision@K + Std-Recall@K)` — F1 using standard recall (supplementary). +- **Golden Match@K**: fraction of queries where DUT Top-K exactly matches the reference golden Top-K. + +The paper uses Hit@K and Precision@K as primary metrics. Std-Recall@K and Std-F1@K are supplementary, +included to show Top-K coverage against all relevant items in the database. + +*Results from Verilator/Cocotb simulation. Not measured on physical FPGA hardware.* + diff --git a/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py b/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py index 54bfed4..2e1b275 100644 --- a/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py +++ b/hw/sim/benchmarks/retrieval/test_retrieval_benchmark.py @@ -288,8 +288,6 @@ async def cam_retrieval_benchmark(dut): write_noise_rate_den = int(get_param(dut, "WRITE_NOISE_RATE_DEN", 100) or 100) mode = mode_from_params(write_noise_en) - if write_noise_en: - raise AssertionError("First retrieval benchmark version only supports WRITE_NOISE_EN=0") if num_rows % lanes != 0: raise AssertionError("Retrieval benchmark requires NUM_ROWS divisible by LANES") @@ -355,4 +353,13 @@ async def cam_retrieval_benchmark(dut): str(out_dir.relative_to(_project_root())), ) - assert result["metrics"]["5"]["exact_match_rate"] == 1.0 + if write_noise_en == 0: + assert result["metrics"]["5"]["exact_match_rate"] == 1.0, ( + f"Expected perfect exact match with no noise, got " + f"{result['metrics']['5']['exact_match_rate']}" + ) + else: + dut._log.info( + "Noise enabled (WRITE_NOISE_RATE=%d/%d) — exact_match assertion skipped", + write_noise_rate_num, write_noise_rate_den, + ) diff --git a/scripts/run_retrieval_noise_sweep.py b/scripts/run_retrieval_noise_sweep.py new file mode 100644 index 0000000..52480d6 --- /dev/null +++ b/scripts/run_retrieval_noise_sweep.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python3 +"""Run CAM retrieval benchmark noise sweep (0%–100%, step 10%) and generate summary. + +Usage: + python scripts/run_retrieval_noise_sweep.py \ + --dataset outputs/cam_retrieval_benchmark/datasets/cifar10_hash512_rows512_queries128.npz \ + --num-rows 512 \ + --output docs/cam_retrieval_noise_sweep.md +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Iterator + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + +DEFAULT_NOISE_RATES = list(range(0, 101, 10)) # 0, 10, 20, ..., 100 + + +@dataclass +class RunResult: + noise_pct: int + run_id: str + metrics: dict # {"1": {...}, "5": {...}} + success: bool + error_msg: str = "" + params: dict = field(default_factory=dict) + dataset_info: dict = field(default_factory=dict) + + +def run_single( + dataset: str, + num_rows: int, + noise_pct: int, + topk_k: int = 5, + hash_bits: int = 512, +) -> RunResult: + """Run a single benchmark with the given noise rate.""" + write_noise_en = 0 if noise_pct == 0 else 1 + # Include dataset stem in run_id to avoid cross-dataset overwrites + dataset_stem = Path(dataset).stem + run_id = f"noise_sweep_{dataset_stem}_{noise_pct:03d}pct" + + env = os.environ.copy() + env["CAM_RETRIEVAL_RUN_ID"] = run_id + env["CAM_RETRIEVAL_DATASET"] = dataset + + make_args = [ + "make", + "-C", "hw/sim", + f"TOPK_K={topk_k}", + f"NUM_ROWS={num_rows}", + f"HASH_BITS={hash_bits}", + f"WRITE_NOISE_EN={write_noise_en}", + ] + if write_noise_en: + make_args.extend([ + f"WRITE_NOISE_RATE_NUM={noise_pct}", + "WRITE_NOISE_RATE_DEN=100", + ]) + + clean_cmd = ["make", "-C", "hw/sim", "clean"] + test_cmd = make_args + ["test-benchmark-retrieval"] + + cwd = str(PROJECT_ROOT) + + # Clean — stream output + print(" [clean]", flush=True) + result_clean = subprocess.run( + clean_cmd, cwd=cwd, env=env, + timeout=120, + ) + if result_clean.returncode != 0: + return RunResult( + noise_pct=noise_pct, + run_id=run_id, + metrics={}, + success=False, + error_msg=f"clean failed (rc={result_clean.returncode})", + ) + + # Run benchmark — stream output but capture stderr for error reporting + print(" [make test-benchmark-retrieval]", flush=True) + try: + result = subprocess.run( + test_cmd, cwd=cwd, env=env, + stderr=subprocess.PIPE, + text=True, + timeout=1800, # 30 min per run + ) + except subprocess.TimeoutExpired: + return RunResult( + noise_pct=noise_pct, + run_id=run_id, + metrics={}, + success=False, + error_msg="test timed out (30 min)", + ) + + if result.returncode != 0: + return RunResult( + noise_pct=noise_pct, + run_id=run_id, + metrics={}, + success=False, + error_msg=f"test failed (rc={result.returncode}):\n{result.stderr[-2000:]}", + ) + + # Read result from output + return _read_result(noise_pct, run_id) + + +def _read_result(noise_pct: int, run_id: str) -> RunResult: + """Read benchmark results from output directory.""" + out_dir = PROJECT_ROOT / "outputs" / "cam_retrieval_benchmark" / run_id + metrics_file = out_dir / "metrics.json" + + if not metrics_file.exists(): + return RunResult( + noise_pct=noise_pct, + run_id=run_id, + metrics={}, + success=False, + error_msg=f"metrics.json not found at {metrics_file}", + ) + + try: + data = json.loads(metrics_file.read_text(encoding="utf-8")) + except Exception as exc: + return RunResult( + noise_pct=noise_pct, + run_id=run_id, + metrics={}, + success=False, + error_msg=f"Failed to parse metrics.json: {exc}", + ) + + return RunResult( + noise_pct=noise_pct, + run_id=run_id, + metrics=data.get("metrics", {}), + success=data.get("status") == "pass", + params=data.get("params", {}), + dataset_info=data.get("dataset", {}), + ) + + +def iter_noise_rates() -> Iterator[int]: + return iter(DEFAULT_NOISE_RATES) + + +def run_sweep( + dataset: str, + num_rows: int, + topk_k: int = 5, + hash_bits: int = 512, +) -> list[RunResult]: + """Run the full noise sweep, return all results.""" + results: list[RunResult] = [] + total = len(DEFAULT_NOISE_RATES) + + for idx, noise_pct in enumerate(DEFAULT_NOISE_RATES): + pct_str = f"{noise_pct:3d}%" + print(f"\n{'='*60}") + print(f" [{idx+1:2d}/{total}] Noise rate: {pct_str}") + print(f"{'='*60}\n", flush=True) + + run_result = run_single( + dataset=dataset, + num_rows=num_rows, + noise_pct=noise_pct, + topk_k=topk_k, + hash_bits=hash_bits, + ) + results.append(run_result) + + if run_result.success: + k1 = run_result.metrics.get("1", {}) + k5 = run_result.metrics.get("5", {}) + print(f" ✓ PASS recall@1={k1.get('recall@k', '?'):.4f} " + f"recall@5={k5.get('recall@k', '?'):.4f} " + f"exact@5={k5.get('exact_match_rate', '?'):.4f}") + else: + print(f" ✗ FAIL {run_result.error_msg[:200]}") + + return results + + +def generate_summary( + results: list[RunResult], + dataset: str, + num_rows: int, + topk_k: int, + hash_bits: int, + output_path: Path, +) -> None: + """Generate comprehensive markdown summary.""" + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + n_total = len(results) + n_pass = sum(1 for r in results if r.success) + n_fail = sum(1 for r in results if not r.success) + + dataset_info = results[0].dataset_info if results else {} + + lines = [ + "# CAM Retrieval Benchmark — Noise Sweep Summary", + "", + f"**Generated:** {now}", + "", + "## Configuration", + "", + "| Parameter | Value |", + "|---|---|", + f"| Dataset path | `{dataset}` |", + f"| NUM_ROWS | {num_rows} |", + f"| TOPK_K | {topk_k} |", + f"| HASH_BITS | {hash_bits} |", + f"| Noise rates | {', '.join(f'{p}%' for p in DEFAULT_NOISE_RATES)} |", + f"| Total runs | {n_total} |", + f"| Passed | {n_pass} |", + f"| Failed | {n_fail} |", + ] + + if dataset_info: + lines.extend([ + "", + "### Dataset Details", + "", + "| Field | Value |", + "|---|---|", + f"| num_queries | {dataset_info.get('num_queries', '?')} |", + f"| num_classes | {dataset_info.get('num_classes', '?')} |", + f"| seed | {dataset_info.get('seed', '?')} |", + ]) + + lines.extend([ + "", + "---", + "", + "## Results by Noise Rate", + "", + ]) + + # Per-rate detailed table + for k in (1, 5): + # Hit@K (hit-rate), Precision@K (mean per-query tp/k), + # Hit-F1@K (F1 from Hit@K × Precision@K), + # Std-Recall@K (mean per-query tp/|relevant|), + # Std-F1@K (F1 from Precision@K × Std-Recall@K), + # Golden Match@K (exact match with reference) + lines.extend([ + f"### k={k}", + "", + "| Noise (%) | WRITE_NOISE_EN | NUM/DEN | Hit@K | Precision@K | Hit-F1@K | Std-Recall@K | Std-F1@K | Golden Match@K | Status |", + "|---|---:|---|---:|---|---:|---|---:|---|---:|---|", + ]) + for r in results: + metrics = r.metrics.get(str(k), {}) + noise_en = r.params.get("write_noise_en", 0) + status = "✓" if r.success else "✗" + if noise_en: + rate_num = r.params.get("write_noise_rate_num", 0) + rate_den = r.params.get("write_noise_rate_den", 100) + rate_str = f"{rate_num}/{rate_den}" + else: + rate_str = "—" + + # Extract raw metrics + hit_k = metrics.get('recall@k', 0) + if isinstance(hit_k, str): hit_k = 0.0 + prec_k = metrics.get('macro_precision', 0) + if isinstance(prec_k, str): prec_k = 0.0 + std_recall = metrics.get('retrieval_recall', 0) + if isinstance(std_recall, str): std_recall = 0.0 + golden = metrics.get('exact_match_rate', 0) + if isinstance(golden, str): golden = 0.0 + + # Hit-F1@K = 2*Hit@K*Precision@K / (Hit@K + Precision@K) + if prec_k + hit_k > 0: + hit_f1 = (2.0 * prec_k * hit_k) / (prec_k + hit_k) + else: + hit_f1 = 0.0 + + # Std-F1@K = 2*Precision@K*Std-Recall@K / (Precision@K + Std-Recall@K) + if prec_k + std_recall > 0: + std_f1 = (2.0 * prec_k * std_recall) / (prec_k + std_recall) + else: + std_f1 = 0.0 + + lines.append( + f"| {r.noise_pct:3d}% | {noise_en} | " + f"{rate_str} | " + f"{hit_k:.6f} | " + f"{prec_k:.6f} | " + f"{hit_f1:.6f} | " + f"{std_recall:.6f} | " + f"{std_f1:.6f} | " + f"{golden:.6f} | " + f"{status} |" + ) + lines.append("") + + # Comparison across noise levels (using primary metric: Hit@K) + lines.extend([ + "---", + "", + "## Cross-Noise Comparison (primary: Hit@K)", + "", + "| Noise (%) | Hit@1 | Hit@5 | Δ(Hit@1 vs 0%) | Δ(Hit@5 vs 0%) |", + "|---|---:|---:|---:|---:|", + ]) + + # Find baseline (0% noise) + zero_result = next((r for r in results if r.noise_pct == 0 and r.success), None) + base_r1 = float(zero_result.metrics.get("1", {}).get("recall@k", 0)) if zero_result else 0.0 + base_r5 = float(zero_result.metrics.get("5", {}).get("recall@k", 0)) if zero_result else 0.0 + + for r in results: + r1 = float(r.metrics.get("1", {}).get("recall@k", 0)) if r.success else float("nan") + r5 = float(r.metrics.get("5", {}).get("recall@k", 0)) if r.success else float("nan") + d1 = f"{r1 - base_r1:+.6f}" if r.success and zero_result else "—" + d5 = f"{r5 - base_r5:+.6f}" if r.success and zero_result else "—" + r1_str = f"{r1:.6f}" if r.success else "FAIL" + r5_str = f"{r5:.6f}" if r.success else "FAIL" + lines.append(f"| {r.noise_pct:3d}% | {r1_str} | {r5_str} | {d1} | {d5} |") + + # Failures section + failures = [r for r in results if not r.success] + if failures: + lines.extend([ + "", + "---", + "", + "## Failed Runs", + "", + ]) + for r in failures: + lines.extend([ + f"### Noise rate: {r.noise_pct}%", + "", + "```", + r.error_msg.strip(), + "```", + "", + ]) + + lines.extend([ + "", + "---", + "", + "## Metric Definitions", + "", + "- **Hit@K**: fraction of queries where at least one relevant item appears in Top-K results (primary metric).", + "- **Precision@K**: mean per-query precision — averaged `tp/k` across all queries.", + "- **Hit-F1@K**: `2 × Hit@K × Precision@K / (Hit@K + Precision@K)` — F1 using hit-rate recall.", + "- **Std-Recall@K**: mean per-query standard retrieval recall — `tp / |relevant|` averaged across queries (supplementary).", + "- **Std-F1@K**: `2 × Precision@K × Std-Recall@K / (Precision@K + Std-Recall@K)` — F1 using standard recall (supplementary).", + "- **Golden Match@K**: fraction of queries where DUT Top-K exactly matches the reference golden Top-K.", + "", + "The paper uses Hit@K and Precision@K as primary metrics. Std-Recall@K and Std-F1@K are supplementary,", + "included to show Top-K coverage against all relevant items in the database.", + "", + "*Results from Verilator/Cocotb simulation. Not measured on physical FPGA hardware.*", + "", + ]) + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"\n Summary written to: {output_path}") + + +def main() -> None: + import argparse + + parser = argparse.ArgumentParser( + description="Run CAM retrieval benchmark noise sweep (0%–100%, step 10%)" + ) + parser.add_argument( + "--dataset", required=True, + help="Path to prepared .npz dataset file", + ) + parser.add_argument( + "--num-rows", type=int, default=512, + help="Number of CAM rows (must match dataset, default: 512)", + ) + parser.add_argument( + "--topk-k", type=int, default=5, + help="TOPK_K parameter (default: 5)", + ) + parser.add_argument( + "--hash-bits", type=int, default=512, + help="HASH_BITS parameter (default: 512)", + ) + parser.add_argument( + "--output", type=Path, default=PROJECT_ROOT / "docs" / "cam_retrieval_noise_sweep.md", + help="Output summary markdown path", + ) + parser.add_argument( + "--noise-rates", + type=str, + default=None, + help="Comma-separated noise rates (default: 0,10,20,...,100)", + ) + args = parser.parse_args() + + # Resolve dataset path + dataset_path = args.dataset + if not os.path.isabs(dataset_path): + dataset_path = str(PROJECT_ROOT / dataset_path) + + # Parse custom noise rates if provided + global DEFAULT_NOISE_RATES + if args.noise_rates: + DEFAULT_NOISE_RATES = [int(x.strip()) for x in args.noise_rates.split(",")] + + print(f"Dataset: {dataset_path}") + print(f"NUM_ROWS: {args.num_rows}") + print(f"TOPK_K: {args.topk_k}") + print(f"HASH_BITS: {args.hash_bits}") + print(f"Noise rates: {DEFAULT_NOISE_RATES}") + print(f"Total runs: {len(DEFAULT_NOISE_RATES)}") + print(f"Output: {args.output}") + print() + + results = run_sweep( + dataset=dataset_path, + num_rows=args.num_rows, + topk_k=args.topk_k, + hash_bits=args.hash_bits, + ) + + generate_summary( + results=results, + dataset=dataset_path, + num_rows=args.num_rows, + topk_k=args.topk_k, + hash_bits=args.hash_bits, + output_path=args.output, + ) + + # Exit with non-zero if any run failed + if any(not r.success for r in results): + sys.exit(1) + + +if __name__ == "__main__": + main()