mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
feat(benchmarks): add noise injection experiment support to CAM retrieval benchmark
- Remove hard assertion blocking WRITE_NOISE_EN=1 in retrieval benchmark tests - Add conditional exact_match assertion: enforces 100% when noise=off, skips when noise=on - New script run_retrieval_noise_sweep.py: sweeps noise 0–100% (step 10%) and produces markdown summary - Add just recipes: cam-benchmark-retrieval-sweep, cam-benchmark-sweep-cifar10, cam-benchmark-sweep-cifar100 - Add rsync-based remote sync commands for outputs and docs
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -221,6 +221,7 @@ openspec/changes/
|
||||
.codegraph/
|
||||
.logs/
|
||||
docs/superpowers
|
||||
.workspace
|
||||
|
||||
# Devenv
|
||||
.devenv*
|
||||
|
||||
27
.justfile
27
.justfile
@@ -97,3 +97,30 @@ cam-test-retrieval-artifact DATASET_PATH NUM_ROWS="4096":
|
||||
# Run CAM retrieval benchmark on a prepared artifact with write noise enabled (Phase 2: read noise removed)
|
||||
cam-test-retrieval-artifact-write-noise DATASET_PATH NUM_ROWS="4096":
|
||||
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 NUM_ROWS={{NUM_ROWS}} WRITE_NOISE_EN=1 WRITE_NOISE_RATE_NUM=1 WRITE_NOISE_RATE_DEN=100 CAM_RETRIEVAL_DATASET={{ DATASET_PATH }}"
|
||||
|
||||
# ── CAM retrieval benchmark noise sweep ────────────────────────────────────────
|
||||
|
||||
# Run noise sweep on a prepared dataset (0%–100%, step 10%)
|
||||
# Usage: just cam-benchmark-retrieval-sweep DATASET=outputs/.../cifar10_hash512_rows512_queries128.npz NUM_ROWS=512
|
||||
cam-benchmark-retrieval-sweep DATASET NUM_ROWS="512":
|
||||
just remote "python scripts/run_retrieval_noise_sweep.py --dataset {{DATASET}} --num-rows {{NUM_ROWS}} --output docs/cam_retrieval_noise_sweep.md"
|
||||
|
||||
# Prepare CIFAR10 dataset + run full noise sweep (all-in-one)
|
||||
cam-benchmark-sweep-cifar10 ROWS="512" QUERIES="128":
|
||||
just cam-prepare-retrieval-cifar10 {{ROWS}} {{QUERIES}}
|
||||
just remote "python scripts/run_retrieval_noise_sweep.py --dataset outputs/cam_retrieval_benchmark/datasets/cifar10_hash512_rows{{ROWS}}_queries{{QUERIES}}.npz --num-rows {{ROWS}} --output docs/cam_retrieval_noise_sweep_cifar10.md"
|
||||
|
||||
# Prepare CIFAR100 dataset + run full noise sweep (all-in-one)
|
||||
cam-benchmark-sweep-cifar100 ROWS="512" QUERIES="128":
|
||||
just cam-prepare-retrieval-cifar100 {{ROWS}} {{QUERIES}}
|
||||
just remote "python scripts/run_retrieval_noise_sweep.py --dataset outputs/cam_retrieval_benchmark/datasets/cifar100_hash512_rows{{ROWS}}_queries{{QUERIES}}.npz --num-rows {{ROWS}} --output docs/cam_retrieval_noise_sweep_cifar100.md"
|
||||
|
||||
# ── Remote ↔ local sync ────────────────────────────────────────────────────────
|
||||
|
||||
# Download benchmark outputs from remote
|
||||
download-outputs:
|
||||
rsync {{ rsync_flags }} {{ remote_root }}/outputs/cam_retrieval_benchmark/ outputs/cam_retrieval_benchmark/
|
||||
|
||||
# Download docs from remote
|
||||
download-docs:
|
||||
rsync {{ rsync_flags }} {{ remote_root }}/docs/ docs/
|
||||
|
||||
95
docs/exps/cam_retrieval_noise_sweep_cifar10.md
Normal file
95
docs/exps/cam_retrieval_noise_sweep_cifar10.md
Normal file
@@ -0,0 +1,95 @@
|
||||
# CAM Retrieval Benchmark — Noise Sweep Summary
|
||||
|
||||
**Generated:** 2026-05-27 19:00:42
|
||||
|
||||
## Configuration
|
||||
|
||||
| Parameter | Value |
|
||||
|---|---|
|
||||
| Dataset path | `outputs/cam_retrieval_benchmark/datasets/cifar10_hash512_rows512_queries128.npz` |
|
||||
| NUM_ROWS | 512 |
|
||||
| TOPK_K | 5 |
|
||||
| HASH_BITS | 512 |
|
||||
| Noise rates | 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100% |
|
||||
| Total runs | 11 |
|
||||
| Passed | 11 |
|
||||
| Failed | 0 |
|
||||
|
||||
### Dataset Details
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| num_queries | 128 |
|
||||
| num_classes | 10 |
|
||||
| seed | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Results by Noise Rate
|
||||
|
||||
### k=1
|
||||
|
||||
| Noise (%) | WRITE_NOISE_EN | NUM/DEN | Hit@K | Precision@K | Hit-F1@K | Std-Recall@K | Std-F1@K | Golden Match@K | Status |
|
||||
|---|---:|---|---:|---|---:|---|---:|---|---:|---|
|
||||
| 0% | 0 | — | 1.000000 | 1.000000 | 1.000000 | 0.019531 | 0.038314 | 1.000000 | ✓ |
|
||||
| 10% | 1 | 10/100 | 1.000000 | 1.000000 | 1.000000 | 0.019531 | 0.038314 | 0.507812 | ✓ |
|
||||
| 20% | 1 | 20/100 | 1.000000 | 1.000000 | 1.000000 | 0.019531 | 0.038314 | 0.234375 | ✓ |
|
||||
| 30% | 1 | 30/100 | 0.992188 | 0.992188 | 0.992188 | 0.019378 | 0.038014 | 0.164062 | ✓ |
|
||||
| 40% | 1 | 40/100 | 0.984375 | 0.984375 | 0.984375 | 0.019228 | 0.037719 | 0.093750 | ✓ |
|
||||
| 50% | 1 | 50/100 | 0.257812 | 0.257812 | 0.257812 | 0.005043 | 0.009893 | 0.023438 | ✓ |
|
||||
| 60% | 1 | 60/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 70% | 1 | 70/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 80% | 1 | 80/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 90% | 1 | 90/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 100% | 1 | 100/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
|
||||
### k=5
|
||||
|
||||
| Noise (%) | WRITE_NOISE_EN | NUM/DEN | Hit@K | Precision@K | Hit-F1@K | Std-Recall@K | Std-F1@K | Golden Match@K | Status |
|
||||
|---|---:|---|---:|---|---:|---|---:|---|---:|---|
|
||||
| 0% | 0 | — | 1.000000 | 1.000000 | 1.000000 | 0.097656 | 0.177936 | 1.000000 | ✓ |
|
||||
| 10% | 1 | 10/100 | 1.000000 | 1.000000 | 1.000000 | 0.097656 | 0.177936 | 0.000000 | ✓ |
|
||||
| 20% | 1 | 20/100 | 1.000000 | 1.000000 | 1.000000 | 0.097656 | 0.177936 | 0.000000 | ✓ |
|
||||
| 30% | 1 | 30/100 | 1.000000 | 0.995313 | 0.997651 | 0.097197 | 0.177099 | 0.000000 | ✓ |
|
||||
| 40% | 1 | 40/100 | 1.000000 | 0.939062 | 0.968574 | 0.091729 | 0.167132 | 0.000000 | ✓ |
|
||||
| 50% | 1 | 50/100 | 0.750000 | 0.234375 | 0.357143 | 0.022913 | 0.041745 | 0.000000 | ✓ |
|
||||
| 60% | 1 | 60/100 | 0.015625 | 0.003125 | 0.005208 | 0.000306 | 0.000558 | 0.000000 | ✓ |
|
||||
| 70% | 1 | 70/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 80% | 1 | 80/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 90% | 1 | 90/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 100% | 1 | 100/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
|
||||
---
|
||||
|
||||
## Cross-Noise Comparison (primary: Hit@K)
|
||||
|
||||
| Noise (%) | Hit@1 | Hit@5 | Δ(Hit@1 vs 0%) | Δ(Hit@5 vs 0%) |
|
||||
|---|---:|---:|---:|---:|
|
||||
| 0% | 1.000000 | 1.000000 | +0.000000 | +0.000000 |
|
||||
| 10% | 1.000000 | 1.000000 | +0.000000 | +0.000000 |
|
||||
| 20% | 1.000000 | 1.000000 | +0.000000 | +0.000000 |
|
||||
| 30% | 0.992188 | 1.000000 | -0.007812 | +0.000000 |
|
||||
| 40% | 0.984375 | 1.000000 | -0.015625 | +0.000000 |
|
||||
| 50% | 0.257812 | 0.750000 | -0.742188 | -0.250000 |
|
||||
| 60% | 0.000000 | 0.015625 | -1.000000 | -0.984375 |
|
||||
| 70% | 0.000000 | 0.000000 | -1.000000 | -1.000000 |
|
||||
| 80% | 0.000000 | 0.000000 | -1.000000 | -1.000000 |
|
||||
| 90% | 0.000000 | 0.000000 | -1.000000 | -1.000000 |
|
||||
| 100% | 0.000000 | 0.000000 | -1.000000 | -1.000000 |
|
||||
|
||||
---
|
||||
|
||||
## Metric Definitions
|
||||
|
||||
- **Hit@K**: fraction of queries where at least one relevant item appears in Top-K results (primary metric).
|
||||
- **Precision@K**: mean per-query precision — averaged `tp/k` across all queries.
|
||||
- **Hit-F1@K**: `2 × Hit@K × Precision@K / (Hit@K + Precision@K)` — F1 using hit-rate recall.
|
||||
- **Std-Recall@K**: mean per-query standard retrieval recall — `tp / |relevant|` averaged across queries (supplementary).
|
||||
- **Std-F1@K**: `2 × Precision@K × Std-Recall@K / (Precision@K + Std-Recall@K)` — F1 using standard recall (supplementary).
|
||||
- **Golden Match@K**: fraction of queries where DUT Top-K exactly matches the reference golden Top-K.
|
||||
|
||||
The paper uses Hit@K and Precision@K as primary metrics. Std-Recall@K and Std-F1@K are supplementary,
|
||||
included to show Top-K coverage against all relevant items in the database.
|
||||
|
||||
*Results from Verilator/Cocotb simulation. Not measured on physical FPGA hardware.*
|
||||
|
||||
95
docs/exps/cam_retrieval_noise_sweep_cifar100.md
Normal file
95
docs/exps/cam_retrieval_noise_sweep_cifar100.md
Normal file
@@ -0,0 +1,95 @@
|
||||
# CAM Retrieval Benchmark — Noise Sweep Summary
|
||||
|
||||
**Generated:** 2026-05-27 19:00:46
|
||||
|
||||
## Configuration
|
||||
|
||||
| Parameter | Value |
|
||||
|---|---|
|
||||
| Dataset path | `outputs/cam_retrieval_benchmark/datasets/cifar100_hash512_rows512_queries128.npz` |
|
||||
| NUM_ROWS | 512 |
|
||||
| TOPK_K | 5 |
|
||||
| HASH_BITS | 512 |
|
||||
| Noise rates | 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100% |
|
||||
| Total runs | 11 |
|
||||
| Passed | 11 |
|
||||
| Failed | 0 |
|
||||
|
||||
### Dataset Details
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| num_queries | 128 |
|
||||
| num_classes | 100 |
|
||||
| seed | 0 |
|
||||
|
||||
---
|
||||
|
||||
## Results by Noise Rate
|
||||
|
||||
### k=1
|
||||
|
||||
| Noise (%) | WRITE_NOISE_EN | NUM/DEN | Hit@K | Precision@K | Hit-F1@K | Std-Recall@K | Std-F1@K | Golden Match@K | Status |
|
||||
|---|---:|---|---:|---|---:|---|---:|---|---:|---|
|
||||
| 0% | 0 | — | 0.695312 | 0.695312 | 0.695312 | 0.134635 | 0.225589 | 1.000000 | ✓ |
|
||||
| 10% | 1 | 10/100 | 0.585938 | 0.585938 | 0.585938 | 0.113281 | 0.189857 | 0.593750 | ✓ |
|
||||
| 20% | 1 | 20/100 | 0.562500 | 0.562500 | 0.562500 | 0.109115 | 0.182774 | 0.460938 | ✓ |
|
||||
| 30% | 1 | 30/100 | 0.460938 | 0.460938 | 0.460938 | 0.088802 | 0.148915 | 0.304688 | ✓ |
|
||||
| 40% | 1 | 40/100 | 0.234375 | 0.234375 | 0.234375 | 0.044792 | 0.075210 | 0.101562 | ✓ |
|
||||
| 50% | 1 | 50/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 60% | 1 | 60/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 70% | 1 | 70/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 80% | 1 | 80/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 90% | 1 | 90/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 100% | 1 | 100/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
|
||||
### k=5
|
||||
|
||||
| Noise (%) | WRITE_NOISE_EN | NUM/DEN | Hit@K | Precision@K | Hit-F1@K | Std-Recall@K | Std-F1@K | Golden Match@K | Status |
|
||||
|---|---:|---|---:|---|---:|---|---:|---|---:|---|
|
||||
| 0% | 0 | — | 0.867188 | 0.462500 | 0.603261 | 0.445052 | 0.453608 | 1.000000 | ✓ |
|
||||
| 10% | 1 | 10/100 | 0.812500 | 0.421875 | 0.555380 | 0.405990 | 0.413780 | 0.023438 | ✓ |
|
||||
| 20% | 1 | 20/100 | 0.742188 | 0.364062 | 0.488502 | 0.350000 | 0.356893 | 0.000000 | ✓ |
|
||||
| 30% | 1 | 30/100 | 0.640625 | 0.248437 | 0.358029 | 0.238802 | 0.243525 | 0.000000 | ✓ |
|
||||
| 40% | 1 | 40/100 | 0.460938 | 0.117187 | 0.186867 | 0.113021 | 0.115066 | 0.000000 | ✓ |
|
||||
| 50% | 1 | 50/100 | 0.062500 | 0.015625 | 0.025000 | 0.014844 | 0.015224 | 0.000000 | ✓ |
|
||||
| 60% | 1 | 60/100 | 0.007812 | 0.001563 | 0.002604 | 0.001563 | 0.001563 | 0.000000 | ✓ |
|
||||
| 70% | 1 | 70/100 | 0.007812 | 0.001563 | 0.002604 | 0.001563 | 0.001563 | 0.000000 | ✓ |
|
||||
| 80% | 1 | 80/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 90% | 1 | 90/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
| 100% | 1 | 100/100 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ✓ |
|
||||
|
||||
---
|
||||
|
||||
## Cross-Noise Comparison (primary: Hit@K)
|
||||
|
||||
| Noise (%) | Hit@1 | Hit@5 | Δ(Hit@1 vs 0%) | Δ(Hit@5 vs 0%) |
|
||||
|---|---:|---:|---:|---:|
|
||||
| 0% | 0.695312 | 0.867188 | +0.000000 | +0.000000 |
|
||||
| 10% | 0.585938 | 0.812500 | -0.109375 | -0.054688 |
|
||||
| 20% | 0.562500 | 0.742188 | -0.132812 | -0.125000 |
|
||||
| 30% | 0.460938 | 0.640625 | -0.234375 | -0.226562 |
|
||||
| 40% | 0.234375 | 0.460938 | -0.460938 | -0.406250 |
|
||||
| 50% | 0.000000 | 0.062500 | -0.695312 | -0.804688 |
|
||||
| 60% | 0.000000 | 0.007812 | -0.695312 | -0.859375 |
|
||||
| 70% | 0.000000 | 0.007812 | -0.695312 | -0.859375 |
|
||||
| 80% | 0.000000 | 0.000000 | -0.695312 | -0.867188 |
|
||||
| 90% | 0.000000 | 0.000000 | -0.695312 | -0.867188 |
|
||||
| 100% | 0.000000 | 0.000000 | -0.695312 | -0.867188 |
|
||||
|
||||
---
|
||||
|
||||
## Metric Definitions
|
||||
|
||||
- **Hit@K**: fraction of queries where at least one relevant item appears in Top-K results (primary metric).
|
||||
- **Precision@K**: mean per-query precision — averaged `tp/k` across all queries.
|
||||
- **Hit-F1@K**: `2 × Hit@K × Precision@K / (Hit@K + Precision@K)` — F1 using hit-rate recall.
|
||||
- **Std-Recall@K**: mean per-query standard retrieval recall — `tp / |relevant|` averaged across queries (supplementary).
|
||||
- **Std-F1@K**: `2 × Precision@K × Std-Recall@K / (Precision@K + Std-Recall@K)` — F1 using standard recall (supplementary).
|
||||
- **Golden Match@K**: fraction of queries where DUT Top-K exactly matches the reference golden Top-K.
|
||||
|
||||
The paper uses Hit@K and Precision@K as primary metrics. Std-Recall@K and Std-F1@K are supplementary,
|
||||
included to show Top-K coverage against all relevant items in the database.
|
||||
|
||||
*Results from Verilator/Cocotb simulation. Not measured on physical FPGA hardware.*
|
||||
|
||||
@@ -288,8 +288,6 @@ async def cam_retrieval_benchmark(dut):
|
||||
write_noise_rate_den = int(get_param(dut, "WRITE_NOISE_RATE_DEN", 100) or 100)
|
||||
mode = mode_from_params(write_noise_en)
|
||||
|
||||
if write_noise_en:
|
||||
raise AssertionError("First retrieval benchmark version only supports WRITE_NOISE_EN=0")
|
||||
if num_rows % lanes != 0:
|
||||
raise AssertionError("Retrieval benchmark requires NUM_ROWS divisible by LANES")
|
||||
|
||||
@@ -355,4 +353,13 @@ async def cam_retrieval_benchmark(dut):
|
||||
str(out_dir.relative_to(_project_root())),
|
||||
)
|
||||
|
||||
assert result["metrics"]["5"]["exact_match_rate"] == 1.0
|
||||
if write_noise_en == 0:
|
||||
assert result["metrics"]["5"]["exact_match_rate"] == 1.0, (
|
||||
f"Expected perfect exact match with no noise, got "
|
||||
f"{result['metrics']['5']['exact_match_rate']}"
|
||||
)
|
||||
else:
|
||||
dut._log.info(
|
||||
"Noise enabled (WRITE_NOISE_RATE=%d/%d) — exact_match assertion skipped",
|
||||
write_noise_rate_num, write_noise_rate_den,
|
||||
)
|
||||
|
||||
454
scripts/run_retrieval_noise_sweep.py
Normal file
454
scripts/run_retrieval_noise_sweep.py
Normal file
@@ -0,0 +1,454 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run CAM retrieval benchmark noise sweep (0%–100%, step 10%) and generate summary.
|
||||
|
||||
Usage:
|
||||
python scripts/run_retrieval_noise_sweep.py \
|
||||
--dataset outputs/cam_retrieval_benchmark/datasets/cifar10_hash512_rows512_queries128.npz \
|
||||
--num-rows 512 \
|
||||
--output docs/cam_retrieval_noise_sweep.md
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
DEFAULT_NOISE_RATES = list(range(0, 101, 10)) # 0, 10, 20, ..., 100
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunResult:
|
||||
noise_pct: int
|
||||
run_id: str
|
||||
metrics: dict # {"1": {...}, "5": {...}}
|
||||
success: bool
|
||||
error_msg: str = ""
|
||||
params: dict = field(default_factory=dict)
|
||||
dataset_info: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
def run_single(
|
||||
dataset: str,
|
||||
num_rows: int,
|
||||
noise_pct: int,
|
||||
topk_k: int = 5,
|
||||
hash_bits: int = 512,
|
||||
) -> RunResult:
|
||||
"""Run a single benchmark with the given noise rate."""
|
||||
write_noise_en = 0 if noise_pct == 0 else 1
|
||||
# Include dataset stem in run_id to avoid cross-dataset overwrites
|
||||
dataset_stem = Path(dataset).stem
|
||||
run_id = f"noise_sweep_{dataset_stem}_{noise_pct:03d}pct"
|
||||
|
||||
env = os.environ.copy()
|
||||
env["CAM_RETRIEVAL_RUN_ID"] = run_id
|
||||
env["CAM_RETRIEVAL_DATASET"] = dataset
|
||||
|
||||
make_args = [
|
||||
"make",
|
||||
"-C", "hw/sim",
|
||||
f"TOPK_K={topk_k}",
|
||||
f"NUM_ROWS={num_rows}",
|
||||
f"HASH_BITS={hash_bits}",
|
||||
f"WRITE_NOISE_EN={write_noise_en}",
|
||||
]
|
||||
if write_noise_en:
|
||||
make_args.extend([
|
||||
f"WRITE_NOISE_RATE_NUM={noise_pct}",
|
||||
"WRITE_NOISE_RATE_DEN=100",
|
||||
])
|
||||
|
||||
clean_cmd = ["make", "-C", "hw/sim", "clean"]
|
||||
test_cmd = make_args + ["test-benchmark-retrieval"]
|
||||
|
||||
cwd = str(PROJECT_ROOT)
|
||||
|
||||
# Clean — stream output
|
||||
print(" [clean]", flush=True)
|
||||
result_clean = subprocess.run(
|
||||
clean_cmd, cwd=cwd, env=env,
|
||||
timeout=120,
|
||||
)
|
||||
if result_clean.returncode != 0:
|
||||
return RunResult(
|
||||
noise_pct=noise_pct,
|
||||
run_id=run_id,
|
||||
metrics={},
|
||||
success=False,
|
||||
error_msg=f"clean failed (rc={result_clean.returncode})",
|
||||
)
|
||||
|
||||
# Run benchmark — stream output but capture stderr for error reporting
|
||||
print(" [make test-benchmark-retrieval]", flush=True)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
test_cmd, cwd=cwd, env=env,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=1800, # 30 min per run
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return RunResult(
|
||||
noise_pct=noise_pct,
|
||||
run_id=run_id,
|
||||
metrics={},
|
||||
success=False,
|
||||
error_msg="test timed out (30 min)",
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
return RunResult(
|
||||
noise_pct=noise_pct,
|
||||
run_id=run_id,
|
||||
metrics={},
|
||||
success=False,
|
||||
error_msg=f"test failed (rc={result.returncode}):\n{result.stderr[-2000:]}",
|
||||
)
|
||||
|
||||
# Read result from output
|
||||
return _read_result(noise_pct, run_id)
|
||||
|
||||
|
||||
def _read_result(noise_pct: int, run_id: str) -> RunResult:
|
||||
"""Read benchmark results from output directory."""
|
||||
out_dir = PROJECT_ROOT / "outputs" / "cam_retrieval_benchmark" / run_id
|
||||
metrics_file = out_dir / "metrics.json"
|
||||
|
||||
if not metrics_file.exists():
|
||||
return RunResult(
|
||||
noise_pct=noise_pct,
|
||||
run_id=run_id,
|
||||
metrics={},
|
||||
success=False,
|
||||
error_msg=f"metrics.json not found at {metrics_file}",
|
||||
)
|
||||
|
||||
try:
|
||||
data = json.loads(metrics_file.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
return RunResult(
|
||||
noise_pct=noise_pct,
|
||||
run_id=run_id,
|
||||
metrics={},
|
||||
success=False,
|
||||
error_msg=f"Failed to parse metrics.json: {exc}",
|
||||
)
|
||||
|
||||
return RunResult(
|
||||
noise_pct=noise_pct,
|
||||
run_id=run_id,
|
||||
metrics=data.get("metrics", {}),
|
||||
success=data.get("status") == "pass",
|
||||
params=data.get("params", {}),
|
||||
dataset_info=data.get("dataset", {}),
|
||||
)
|
||||
|
||||
|
||||
def iter_noise_rates() -> Iterator[int]:
|
||||
return iter(DEFAULT_NOISE_RATES)
|
||||
|
||||
|
||||
def run_sweep(
|
||||
dataset: str,
|
||||
num_rows: int,
|
||||
topk_k: int = 5,
|
||||
hash_bits: int = 512,
|
||||
) -> list[RunResult]:
|
||||
"""Run the full noise sweep, return all results."""
|
||||
results: list[RunResult] = []
|
||||
total = len(DEFAULT_NOISE_RATES)
|
||||
|
||||
for idx, noise_pct in enumerate(DEFAULT_NOISE_RATES):
|
||||
pct_str = f"{noise_pct:3d}%"
|
||||
print(f"\n{'='*60}")
|
||||
print(f" [{idx+1:2d}/{total}] Noise rate: {pct_str}")
|
||||
print(f"{'='*60}\n", flush=True)
|
||||
|
||||
run_result = run_single(
|
||||
dataset=dataset,
|
||||
num_rows=num_rows,
|
||||
noise_pct=noise_pct,
|
||||
topk_k=topk_k,
|
||||
hash_bits=hash_bits,
|
||||
)
|
||||
results.append(run_result)
|
||||
|
||||
if run_result.success:
|
||||
k1 = run_result.metrics.get("1", {})
|
||||
k5 = run_result.metrics.get("5", {})
|
||||
print(f" ✓ PASS recall@1={k1.get('recall@k', '?'):.4f} "
|
||||
f"recall@5={k5.get('recall@k', '?'):.4f} "
|
||||
f"exact@5={k5.get('exact_match_rate', '?'):.4f}")
|
||||
else:
|
||||
print(f" ✗ FAIL {run_result.error_msg[:200]}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def generate_summary(
|
||||
results: list[RunResult],
|
||||
dataset: str,
|
||||
num_rows: int,
|
||||
topk_k: int,
|
||||
hash_bits: int,
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
"""Generate comprehensive markdown summary."""
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
n_total = len(results)
|
||||
n_pass = sum(1 for r in results if r.success)
|
||||
n_fail = sum(1 for r in results if not r.success)
|
||||
|
||||
dataset_info = results[0].dataset_info if results else {}
|
||||
|
||||
lines = [
|
||||
"# CAM Retrieval Benchmark — Noise Sweep Summary",
|
||||
"",
|
||||
f"**Generated:** {now}",
|
||||
"",
|
||||
"## Configuration",
|
||||
"",
|
||||
"| Parameter | Value |",
|
||||
"|---|---|",
|
||||
f"| Dataset path | `{dataset}` |",
|
||||
f"| NUM_ROWS | {num_rows} |",
|
||||
f"| TOPK_K | {topk_k} |",
|
||||
f"| HASH_BITS | {hash_bits} |",
|
||||
f"| Noise rates | {', '.join(f'{p}%' for p in DEFAULT_NOISE_RATES)} |",
|
||||
f"| Total runs | {n_total} |",
|
||||
f"| Passed | {n_pass} |",
|
||||
f"| Failed | {n_fail} |",
|
||||
]
|
||||
|
||||
if dataset_info:
|
||||
lines.extend([
|
||||
"",
|
||||
"### Dataset Details",
|
||||
"",
|
||||
"| Field | Value |",
|
||||
"|---|---|",
|
||||
f"| num_queries | {dataset_info.get('num_queries', '?')} |",
|
||||
f"| num_classes | {dataset_info.get('num_classes', '?')} |",
|
||||
f"| seed | {dataset_info.get('seed', '?')} |",
|
||||
])
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"## Results by Noise Rate",
|
||||
"",
|
||||
])
|
||||
|
||||
# Per-rate detailed table
|
||||
for k in (1, 5):
|
||||
# Hit@K (hit-rate), Precision@K (mean per-query tp/k),
|
||||
# Hit-F1@K (F1 from Hit@K × Precision@K),
|
||||
# Std-Recall@K (mean per-query tp/|relevant|),
|
||||
# Std-F1@K (F1 from Precision@K × Std-Recall@K),
|
||||
# Golden Match@K (exact match with reference)
|
||||
lines.extend([
|
||||
f"### k={k}",
|
||||
"",
|
||||
"| Noise (%) | WRITE_NOISE_EN | NUM/DEN | Hit@K | Precision@K | Hit-F1@K | Std-Recall@K | Std-F1@K | Golden Match@K | Status |",
|
||||
"|---|---:|---|---:|---|---:|---|---:|---|---:|---|",
|
||||
])
|
||||
for r in results:
|
||||
metrics = r.metrics.get(str(k), {})
|
||||
noise_en = r.params.get("write_noise_en", 0)
|
||||
status = "✓" if r.success else "✗"
|
||||
if noise_en:
|
||||
rate_num = r.params.get("write_noise_rate_num", 0)
|
||||
rate_den = r.params.get("write_noise_rate_den", 100)
|
||||
rate_str = f"{rate_num}/{rate_den}"
|
||||
else:
|
||||
rate_str = "—"
|
||||
|
||||
# Extract raw metrics
|
||||
hit_k = metrics.get('recall@k', 0)
|
||||
if isinstance(hit_k, str): hit_k = 0.0
|
||||
prec_k = metrics.get('macro_precision', 0)
|
||||
if isinstance(prec_k, str): prec_k = 0.0
|
||||
std_recall = metrics.get('retrieval_recall', 0)
|
||||
if isinstance(std_recall, str): std_recall = 0.0
|
||||
golden = metrics.get('exact_match_rate', 0)
|
||||
if isinstance(golden, str): golden = 0.0
|
||||
|
||||
# Hit-F1@K = 2*Hit@K*Precision@K / (Hit@K + Precision@K)
|
||||
if prec_k + hit_k > 0:
|
||||
hit_f1 = (2.0 * prec_k * hit_k) / (prec_k + hit_k)
|
||||
else:
|
||||
hit_f1 = 0.0
|
||||
|
||||
# Std-F1@K = 2*Precision@K*Std-Recall@K / (Precision@K + Std-Recall@K)
|
||||
if prec_k + std_recall > 0:
|
||||
std_f1 = (2.0 * prec_k * std_recall) / (prec_k + std_recall)
|
||||
else:
|
||||
std_f1 = 0.0
|
||||
|
||||
lines.append(
|
||||
f"| {r.noise_pct:3d}% | {noise_en} | "
|
||||
f"{rate_str} | "
|
||||
f"{hit_k:.6f} | "
|
||||
f"{prec_k:.6f} | "
|
||||
f"{hit_f1:.6f} | "
|
||||
f"{std_recall:.6f} | "
|
||||
f"{std_f1:.6f} | "
|
||||
f"{golden:.6f} | "
|
||||
f"{status} |"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Comparison across noise levels (using primary metric: Hit@K)
|
||||
lines.extend([
|
||||
"---",
|
||||
"",
|
||||
"## Cross-Noise Comparison (primary: Hit@K)",
|
||||
"",
|
||||
"| Noise (%) | Hit@1 | Hit@5 | Δ(Hit@1 vs 0%) | Δ(Hit@5 vs 0%) |",
|
||||
"|---|---:|---:|---:|---:|",
|
||||
])
|
||||
|
||||
# Find baseline (0% noise)
|
||||
zero_result = next((r for r in results if r.noise_pct == 0 and r.success), None)
|
||||
base_r1 = float(zero_result.metrics.get("1", {}).get("recall@k", 0)) if zero_result else 0.0
|
||||
base_r5 = float(zero_result.metrics.get("5", {}).get("recall@k", 0)) if zero_result else 0.0
|
||||
|
||||
for r in results:
|
||||
r1 = float(r.metrics.get("1", {}).get("recall@k", 0)) if r.success else float("nan")
|
||||
r5 = float(r.metrics.get("5", {}).get("recall@k", 0)) if r.success else float("nan")
|
||||
d1 = f"{r1 - base_r1:+.6f}" if r.success and zero_result else "—"
|
||||
d5 = f"{r5 - base_r5:+.6f}" if r.success and zero_result else "—"
|
||||
r1_str = f"{r1:.6f}" if r.success else "FAIL"
|
||||
r5_str = f"{r5:.6f}" if r.success else "FAIL"
|
||||
lines.append(f"| {r.noise_pct:3d}% | {r1_str} | {r5_str} | {d1} | {d5} |")
|
||||
|
||||
# Failures section
|
||||
failures = [r for r in results if not r.success]
|
||||
if failures:
|
||||
lines.extend([
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"## Failed Runs",
|
||||
"",
|
||||
])
|
||||
for r in failures:
|
||||
lines.extend([
|
||||
f"### Noise rate: {r.noise_pct}%",
|
||||
"",
|
||||
"```",
|
||||
r.error_msg.strip(),
|
||||
"```",
|
||||
"",
|
||||
])
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"## Metric Definitions",
|
||||
"",
|
||||
"- **Hit@K**: fraction of queries where at least one relevant item appears in Top-K results (primary metric).",
|
||||
"- **Precision@K**: mean per-query precision — averaged `tp/k` across all queries.",
|
||||
"- **Hit-F1@K**: `2 × Hit@K × Precision@K / (Hit@K + Precision@K)` — F1 using hit-rate recall.",
|
||||
"- **Std-Recall@K**: mean per-query standard retrieval recall — `tp / |relevant|` averaged across queries (supplementary).",
|
||||
"- **Std-F1@K**: `2 × Precision@K × Std-Recall@K / (Precision@K + Std-Recall@K)` — F1 using standard recall (supplementary).",
|
||||
"- **Golden Match@K**: fraction of queries where DUT Top-K exactly matches the reference golden Top-K.",
|
||||
"",
|
||||
"The paper uses Hit@K and Precision@K as primary metrics. Std-Recall@K and Std-F1@K are supplementary,",
|
||||
"included to show Top-K coverage against all relevant items in the database.",
|
||||
"",
|
||||
"*Results from Verilator/Cocotb simulation. Not measured on physical FPGA hardware.*",
|
||||
"",
|
||||
])
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print(f"\n Summary written to: {output_path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run CAM retrieval benchmark noise sweep (0%–100%, step 10%)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dataset", required=True,
|
||||
help="Path to prepared .npz dataset file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-rows", type=int, default=512,
|
||||
help="Number of CAM rows (must match dataset, default: 512)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--topk-k", type=int, default=5,
|
||||
help="TOPK_K parameter (default: 5)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hash-bits", type=int, default=512,
|
||||
help="HASH_BITS parameter (default: 512)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", type=Path, default=PROJECT_ROOT / "docs" / "cam_retrieval_noise_sweep.md",
|
||||
help="Output summary markdown path",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--noise-rates",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Comma-separated noise rates (default: 0,10,20,...,100)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Resolve dataset path
|
||||
dataset_path = args.dataset
|
||||
if not os.path.isabs(dataset_path):
|
||||
dataset_path = str(PROJECT_ROOT / dataset_path)
|
||||
|
||||
# Parse custom noise rates if provided
|
||||
global DEFAULT_NOISE_RATES
|
||||
if args.noise_rates:
|
||||
DEFAULT_NOISE_RATES = [int(x.strip()) for x in args.noise_rates.split(",")]
|
||||
|
||||
print(f"Dataset: {dataset_path}")
|
||||
print(f"NUM_ROWS: {args.num_rows}")
|
||||
print(f"TOPK_K: {args.topk_k}")
|
||||
print(f"HASH_BITS: {args.hash_bits}")
|
||||
print(f"Noise rates: {DEFAULT_NOISE_RATES}")
|
||||
print(f"Total runs: {len(DEFAULT_NOISE_RATES)}")
|
||||
print(f"Output: {args.output}")
|
||||
print()
|
||||
|
||||
results = run_sweep(
|
||||
dataset=dataset_path,
|
||||
num_rows=args.num_rows,
|
||||
topk_k=args.topk_k,
|
||||
hash_bits=args.hash_bits,
|
||||
)
|
||||
|
||||
generate_summary(
|
||||
results=results,
|
||||
dataset=dataset_path,
|
||||
num_rows=args.num_rows,
|
||||
topk_k=args.topk_k,
|
||||
hash_bits=args.hash_bits,
|
||||
output_path=args.output,
|
||||
)
|
||||
|
||||
# Exit with non-zero if any run failed
|
||||
if any(not r.success for r in results):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user