SikongJueluo/Mini-Nav
1
0
Fork 0
mirror of https://github.com/SikongJueluo/Mini-Nav.git synced 2026-07-12 20:15:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(benchmark): support multi-dataset evaluation with configurable top-k list

Browse Source 
- Evaluate multiple datasets in a single run (CIFAR10 + CIFAR100)
- Report Recall@K for a list of K values from one underlying search
- Save results to disk: summary.md, metrics.csv, per-dataset predictions.csv, confusion_matrix.csv
- Richer evaluation output: query_labels, topk_ids, topk_labels for downstream analysis
- Add --dataset and --top-k CLI overrides for benchmark command
- Update config schema: dataset→datasets, top_k→top_k_list
This commit is contained in:
SikongJueluo
2026-05-31 15:56:35 +08:00
parent 7a1e1ccf3f
commit 9eb52f8cef
6 changed files with 454 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
.justfile
View File

@@ -25,8 +25,12 @@ ssh:
-L 127.0.0.1:2718:172.30.0.2:2718 \
{{ remote_ssh_target }}
docker:
docker exec -it {{ remote_docker_container }} bash
docker cmd="":
@if [ -z "{{ cmd }}" ]; then \
docker exec -it -w {{ env("REMOTE_WORKDIR") }} {{ remote_docker_container }} bash; \
else \
docker exec -i -w {{ env("REMOTE_WORKDIR") }} {{ remote_docker_container }} bash -lc {{ quote(cmd) }}; \
fi
marimo +notebook:
uv run marimo edit {{ notebook }} --host 0.0.0.0 --port 2718 --no-token
@@ -84,36 +88,36 @@ cam-test-retrieval-write-noise:
# Prepare CIFAR10 hash artifact for CAM retrieval smoke benchmark
cam-prepare-retrieval-cifar10 ROWS="512" QUERIES="128":
just remote "python scripts/prepare_cam_retrieval_dataset.py --dataset cifar10 --num-rows {{ROWS}} --max-queries {{QUERIES}} --compressor-path outputs/hash_compressor.pt"
just remote "python scripts/prepare_cam_retrieval_dataset.py --dataset cifar10 --num-rows {{ ROWS }} --max-queries {{ QUERIES }} --compressor-path outputs/hash_compressor.pt"
# Prepare CIFAR100 hash artifact for CAM retrieval benchmark
cam-prepare-retrieval-cifar100 ROWS="512" QUERIES="128":
just remote "python scripts/prepare_cam_retrieval_dataset.py --dataset cifar100 --num-rows {{ROWS}} --max-queries {{QUERIES}} --compressor-path outputs/hash_compressor.pt"
just remote "python scripts/prepare_cam_retrieval_dataset.py --dataset cifar100 --num-rows {{ ROWS }} --max-queries {{ QUERIES }} --compressor-path outputs/hash_compressor.pt"
# Run CAM retrieval benchmark on a prepared artifact without hardware noise
cam-test-retrieval-artifact DATASET_PATH NUM_ROWS="4096":
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 NUM_ROWS={{NUM_ROWS}} WRITE_NOISE_EN=0 CAM_RETRIEVAL_DATASET={{ DATASET_PATH }}"
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 NUM_ROWS={{ NUM_ROWS }} WRITE_NOISE_EN=0 CAM_RETRIEVAL_DATASET={{ DATASET_PATH }}"
# Run CAM retrieval benchmark on a prepared artifact with write noise enabled (Phase 2: read noise removed)
cam-test-retrieval-artifact-write-noise DATASET_PATH NUM_ROWS="4096":
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 NUM_ROWS={{NUM_ROWS}} WRITE_NOISE_EN=1 WRITE_NOISE_RATE_NUM=1 WRITE_NOISE_RATE_DEN=100 CAM_RETRIEVAL_DATASET={{ DATASET_PATH }}"
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 NUM_ROWS={{ NUM_ROWS }} WRITE_NOISE_EN=1 WRITE_NOISE_RATE_NUM=1 WRITE_NOISE_RATE_DEN=100 CAM_RETRIEVAL_DATASET={{ DATASET_PATH }}"
# ── CAM retrieval benchmark noise sweep ────────────────────────────────────────
# Run noise sweep on a prepared dataset (0%–100%, step 10%)
# Usage: just cam-benchmark-retrieval-sweep DATASET=outputs/.../cifar10_hash512_rows512_queries128.npz NUM_ROWS=512
cam-benchmark-retrieval-sweep DATASET NUM_ROWS="512":
just remote "python scripts/run_retrieval_noise_sweep.py --dataset {{DATASET}} --num-rows {{NUM_ROWS}} --output docs/cam_retrieval_noise_sweep.md"
just remote "python scripts/run_retrieval_noise_sweep.py --dataset {{ DATASET }} --num-rows {{ NUM_ROWS }} --output docs/cam_retrieval_noise_sweep.md"
# Prepare CIFAR10 dataset + run full noise sweep (all-in-one)
cam-benchmark-sweep-cifar10 ROWS="512" QUERIES="128":
just cam-prepare-retrieval-cifar10 {{ROWS}} {{QUERIES}}
just remote "python scripts/run_retrieval_noise_sweep.py --dataset outputs/cam_retrieval_benchmark/datasets/cifar10_hash512_rows{{ROWS}}_queries{{QUERIES}}.npz --num-rows {{ROWS}} --output docs/cam_retrieval_noise_sweep_cifar10.md"
just cam-prepare-retrieval-cifar10 {{ ROWS }} {{ QUERIES }}
just remote "python scripts/run_retrieval_noise_sweep.py --dataset outputs/cam_retrieval_benchmark/datasets/cifar10_hash512_rows{{ ROWS }}_queries{{ QUERIES }}.npz --num-rows {{ ROWS }} --output docs/cam_retrieval_noise_sweep_cifar10.md"
# Prepare CIFAR100 dataset + run full noise sweep (all-in-one)
cam-benchmark-sweep-cifar100 ROWS="512" QUERIES="128":
just cam-prepare-retrieval-cifar100 {{ROWS}} {{QUERIES}}
just remote "python scripts/run_retrieval_noise_sweep.py --dataset outputs/cam_retrieval_benchmark/datasets/cifar100_hash512_rows{{ROWS}}_queries{{QUERIES}}.npz --num-rows {{ROWS}} --output docs/cam_retrieval_noise_sweep_cifar100.md"
just cam-prepare-retrieval-cifar100 {{ ROWS }} {{ QUERIES }}
just remote "python scripts/run_retrieval_noise_sweep.py --dataset outputs/cam_retrieval_benchmark/datasets/cifar100_hash512_rows{{ ROWS }}_queries{{ QUERIES }}.npz --num-rows {{ ROWS }} --output docs/cam_retrieval_noise_sweep_cifar100.md"
# ── Remote ↔ local sync ────────────────────────────────────────────────────────