mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
feat(benchmark): add software CAM retrieval benchmark
Add software-based CAM retrieval benchmark to compare retrieval quality and speed against hardware simulation. Includes experiment documentation with noise sweep analysis on CIFAR-10/100 datasets. - Add sw_retrieval_benchmark.py for software Hamming distance Top-K retrieval - Add test_sw_retrieval_benchmark.py with unit tests for dataset loading and metrics - Add experiment doc (sw_hw_cam_retrieval_benchmark.md) comparing software vs hardware - Document noise sweep impact on retrieval quality at various WRITE_NOISE_RATE values
This commit is contained in:
152
tests/test_sw_retrieval_benchmark.py
Normal file
152
tests/test_sw_retrieval_benchmark.py
Normal file
@@ -0,0 +1,152 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
|
||||
def load_sw_benchmark():
|
||||
path = Path(__file__).resolve().parent.parent / "scripts" / "sw_retrieval_benchmark.py"
|
||||
spec = importlib.util.spec_from_file_location("sw_retrieval_benchmark", path)
|
||||
assert spec is not None, f"Could not find spec for {path}"
|
||||
assert spec.loader is not None, f"No loader available for {path}"
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _write_dataset(path: Path) -> None:
|
||||
np.savez_compressed(
|
||||
path,
|
||||
rows_words=np.array(
|
||||
[
|
||||
[0b11110000],
|
||||
[0b11100000],
|
||||
[0b00001111],
|
||||
],
|
||||
dtype=np.uint64,
|
||||
),
|
||||
row_labels=np.array([0, 0, 1], dtype=np.int64),
|
||||
queries_words=np.array(
|
||||
[
|
||||
[0b11110000],
|
||||
[0b00001111],
|
||||
],
|
||||
dtype=np.uint64,
|
||||
),
|
||||
query_labels=np.array([0, 1], dtype=np.int64),
|
||||
)
|
||||
|
||||
|
||||
def test_load_dataset_preserves_little_endian_words(tmp_path):
|
||||
bench = load_sw_benchmark()
|
||||
dataset_path = tmp_path / "dataset.npz"
|
||||
np.savez_compressed(
|
||||
dataset_path,
|
||||
rows_words=np.array([[0x2, 0x1]], dtype=np.uint64),
|
||||
row_labels=np.array([7], dtype=np.int64),
|
||||
queries_words=np.array([[0x4, 0x3]], dtype=np.uint64),
|
||||
query_labels=np.array([8], dtype=np.int64),
|
||||
)
|
||||
|
||||
dataset = bench.load_retrieval_dataset_npz(dataset_path)
|
||||
|
||||
assert dataset.rows == [0x1_0000_0000_0000_0002]
|
||||
assert dataset.queries == [0x3_0000_0000_0000_0004]
|
||||
assert dataset.row_labels == [7]
|
||||
assert dataset.query_labels == [8]
|
||||
|
||||
|
||||
def test_run_benchmark_reports_quality_and_query_speed(tmp_path):
|
||||
bench = load_sw_benchmark()
|
||||
dataset_path = tmp_path / "dataset.npz"
|
||||
_write_dataset(dataset_path)
|
||||
timer_values = iter([1_000, 3_000])
|
||||
|
||||
result = bench.run_benchmark(
|
||||
dataset_path,
|
||||
hash_bits=64,
|
||||
topk_values=(1, 2),
|
||||
run_id="unit-test",
|
||||
timer_ns=lambda: next(timer_values),
|
||||
)
|
||||
|
||||
assert result["mode"] == "software-hamming"
|
||||
assert result["status"] == "pass"
|
||||
assert result["dataset"]["num_queries"] == 2
|
||||
assert result["params"]["num_rows"] == 3
|
||||
assert result["params"]["topk_k"] == 2
|
||||
assert result["metrics"]["1"]["exact_match_rate"] == 1.0
|
||||
assert result["metrics"]["1"]["recall@k"] == 1.0
|
||||
assert result["performance"]["total_elapsed_ns"] == 2_000
|
||||
assert result["performance"]["ns_per_query"] == 1_000.0
|
||||
assert result["performance"]["queries_per_second"] == 1_000_000.0
|
||||
|
||||
|
||||
def test_run_benchmark_exact_match_compares_against_reference(tmp_path, monkeypatch):
|
||||
bench = load_sw_benchmark()
|
||||
dataset_path = tmp_path / "dataset.npz"
|
||||
_write_dataset(dataset_path)
|
||||
|
||||
monkeypatch.setattr(bench, "ref_match_topk", lambda query, rows, width, k: ([1, 0], []))
|
||||
|
||||
result = bench.run_benchmark(
|
||||
dataset_path,
|
||||
hash_bits=64,
|
||||
topk_values=(1,),
|
||||
timer_ns=lambda: 0,
|
||||
)
|
||||
|
||||
assert result["metrics"]["1"]["exact_match_rate"] == 0.0
|
||||
|
||||
|
||||
def test_run_benchmark_rejects_hash_bits_that_do_not_match_npz_width(tmp_path):
|
||||
bench = load_sw_benchmark()
|
||||
dataset_path = tmp_path / "dataset.npz"
|
||||
_write_dataset(dataset_path)
|
||||
|
||||
with pytest.raises(ValueError, match="hash_bits"):
|
||||
bench.run_benchmark(dataset_path, hash_bits=8)
|
||||
|
||||
|
||||
def test_write_outputs_includes_quality_and_performance_csv(tmp_path):
|
||||
bench = load_sw_benchmark()
|
||||
result = {
|
||||
"run_id": "unit-test",
|
||||
"mode": "software-hamming",
|
||||
"status": "pass",
|
||||
"params": {"num_rows": 3, "hash_bits": 64, "topk_k": 2},
|
||||
"dataset": {"num_classes": 2, "num_queries": 2},
|
||||
"metrics": {
|
||||
"1": {
|
||||
"macro_precision": 1.0,
|
||||
"retrieval_recall": 0.75,
|
||||
"macro_f1": 0.85,
|
||||
"exact_match_rate": 1.0,
|
||||
"recall@k": 1.0,
|
||||
}
|
||||
},
|
||||
"performance": {
|
||||
"total_elapsed_ns": 2_000,
|
||||
"total_elapsed_sec": 0.000002,
|
||||
"ns_per_query": 1_000.0,
|
||||
"queries_per_second": 1_000_000.0,
|
||||
},
|
||||
}
|
||||
|
||||
bench.write_outputs(tmp_path, result)
|
||||
|
||||
metrics_json = json.loads((tmp_path / "metrics.json").read_text(encoding="utf-8"))
|
||||
metrics_csv = (tmp_path / "metrics.csv").read_text(encoding="utf-8")
|
||||
summary_md = (tmp_path / "summary.md").read_text(encoding="utf-8")
|
||||
|
||||
assert metrics_json["performance"]["queries_per_second"] == 1_000_000.0
|
||||
assert "queries_per_second" in metrics_csv
|
||||
assert "1000000.0" in metrics_csv
|
||||
assert "Software CAM Retrieval Benchmark Summary" in summary_md
|
||||
assert "queries_per_second" in summary_md
|
||||
Reference in New Issue
Block a user