mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
SikongJueluo
9eb52f8cef
- Evaluate multiple datasets in a single run (CIFAR10 + CIFAR100) - Report Recall@K for a list of K values from one underlying search - Save results to disk: summary.md, metrics.csv, per-dataset predictions.csv, confusion_matrix.csv - Richer evaluation output: query_labels, topk_ids, topk_labels for downstream analysis - Add --dataset and --top-k CLI overrides for benchmark command - Update config schema: dataset→datasets, top_k→top_k_list
131 lines
6.2 KiB
Makefile
131 lines
6.2 KiB
Makefile
set dotenv-required := true
|
||
|
||
export MSYS2_ARG_CONV_EXCL := "*"
|
||
export MSYS2_ENV_CONV_EXCL := "*"
|
||
remote_ssh_target := env("REMOTE_SSH_TARGET")
|
||
remote_docker_container := env("REMOTE_DOCKER_CONTAINER")
|
||
remote_root := "$REMOTE_SSH_TARGET:$REMOTE_WORKDIR"
|
||
rsync_flags := "-avLh --progress --stats --itemize-changes"
|
||
upload_excludes := "--exclude-from=.stignore"
|
||
|
||
upload:
|
||
rsync {{ rsync_flags }} {{ upload_excludes }} . {{ remote_root }}/; \
|
||
|
||
sync-pkgs:
|
||
uv sync --inexact
|
||
|
||
sync-data:
|
||
python -m habitat_sim.utils.datasets_download --uids habitat_test_scenes --data-path data/
|
||
python -m habitat_sim.utils.datasets_download --uids habitat_example_objects --data-path data/
|
||
|
||
ssh:
|
||
ssh \
|
||
-L 127.0.0.1:8385:127.0.0.1:42705 \
|
||
-L 127.0.0.1:9098:127.0.0.1:9098 \
|
||
-L 127.0.0.1:2718:172.30.0.2:2718 \
|
||
{{ remote_ssh_target }}
|
||
|
||
docker cmd="":
|
||
@if [ -z "{{ cmd }}" ]; then \
|
||
docker exec -it -w {{ env("REMOTE_WORKDIR") }} {{ remote_docker_container }} bash; \
|
||
else \
|
||
docker exec -i -w {{ env("REMOTE_WORKDIR") }} {{ remote_docker_container }} bash -lc {{ quote(cmd) }}; \
|
||
fi
|
||
|
||
marimo +notebook:
|
||
uv run marimo edit {{ notebook }} --host 0.0.0.0 --port 2718 --no-token
|
||
|
||
add +packages:
|
||
uv add {{ packages }} --no-sync
|
||
just sync-pkgs
|
||
|
||
remove +packages:
|
||
uv remove {{ packages }} --no-sync
|
||
just sync-pkgs
|
||
|
||
remote cmd:
|
||
uv run python scripts/remote_docker_run.py run -- {{ quote(cmd) }}
|
||
|
||
remote-check:
|
||
uv run python scripts/remote_docker_run.py check
|
||
|
||
remote-dry cmd:
|
||
uv run python scripts/remote_docker_run.py run --dry-run -- {{ quote(cmd) }}
|
||
|
||
# ── CAM verification ──────────────────────────────────────────────────────────
|
||
|
||
# Run all CAM tests in one pass (model + top + all modules)
|
||
cam-test-full:
|
||
just remote "make -C hw/sim clean && make -C hw/sim test-all"
|
||
|
||
# Run top CAM tests
|
||
cam-test-top:
|
||
just remote "make -C hw/sim clean && make -C hw/sim test-top"
|
||
|
||
# Run full-scale CAM Yosys synthesis and resource reporting
|
||
cam-synth:
|
||
make -C hw/syn synth
|
||
|
||
# Run Python reference model tests
|
||
cam-test-py:
|
||
make -C hw/sim test-model
|
||
|
||
# Run a specific CAM module test (e.g., cam-test-module MODULE=cam_core_banked)
|
||
cam-test-module MODULE:
|
||
just remote "make -C hw/sim clean && make -C hw/sim test-module MODULE={{ MODULE }}"
|
||
|
||
# Run a single test case within a module (e.g., cam-test MODULE=cam_core_banked TESTCASE=banked_core_reads_aligned_eight_row_batch_after_one_cycle)
|
||
cam-test MODULE TESTCASE:
|
||
just remote "make -C hw/sim clean && make -C hw/sim test-module MODULE={{ MODULE }} COCOTB_TESTCASE={{ TESTCASE }}"
|
||
|
||
# Run CAM retrieval benchmark without hardware noise
|
||
cam-test-retrieval-no-noise:
|
||
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 WRITE_NOISE_EN=0"
|
||
|
||
# Run CAM retrieval benchmark with write noise enabled (Phase 2: read noise removed)
|
||
cam-test-retrieval-write-noise:
|
||
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 WRITE_NOISE_EN=1 WRITE_NOISE_RATE_NUM=1 WRITE_NOISE_RATE_DEN=100"
|
||
|
||
# Prepare CIFAR10 hash artifact for CAM retrieval smoke benchmark
|
||
cam-prepare-retrieval-cifar10 ROWS="512" QUERIES="128":
|
||
just remote "python scripts/prepare_cam_retrieval_dataset.py --dataset cifar10 --num-rows {{ ROWS }} --max-queries {{ QUERIES }} --compressor-path outputs/hash_compressor.pt"
|
||
|
||
# Prepare CIFAR100 hash artifact for CAM retrieval benchmark
|
||
cam-prepare-retrieval-cifar100 ROWS="512" QUERIES="128":
|
||
just remote "python scripts/prepare_cam_retrieval_dataset.py --dataset cifar100 --num-rows {{ ROWS }} --max-queries {{ QUERIES }} --compressor-path outputs/hash_compressor.pt"
|
||
|
||
# Run CAM retrieval benchmark on a prepared artifact without hardware noise
|
||
cam-test-retrieval-artifact DATASET_PATH NUM_ROWS="4096":
|
||
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 NUM_ROWS={{ NUM_ROWS }} WRITE_NOISE_EN=0 CAM_RETRIEVAL_DATASET={{ DATASET_PATH }}"
|
||
|
||
# Run CAM retrieval benchmark on a prepared artifact with write noise enabled (Phase 2: read noise removed)
|
||
cam-test-retrieval-artifact-write-noise DATASET_PATH NUM_ROWS="4096":
|
||
just remote "make -C hw/sim clean && make -C hw/sim test-benchmark-retrieval TOPK_K=5 NUM_ROWS={{ NUM_ROWS }} WRITE_NOISE_EN=1 WRITE_NOISE_RATE_NUM=1 WRITE_NOISE_RATE_DEN=100 CAM_RETRIEVAL_DATASET={{ DATASET_PATH }}"
|
||
|
||
# ── CAM retrieval benchmark noise sweep ────────────────────────────────────────
|
||
# Run noise sweep on a prepared dataset (0%–100%, step 10%)
|
||
|
||
# Usage: just cam-benchmark-retrieval-sweep DATASET=outputs/.../cifar10_hash512_rows512_queries128.npz NUM_ROWS=512
|
||
cam-benchmark-retrieval-sweep DATASET NUM_ROWS="512":
|
||
just remote "python scripts/run_retrieval_noise_sweep.py --dataset {{ DATASET }} --num-rows {{ NUM_ROWS }} --output docs/cam_retrieval_noise_sweep.md"
|
||
|
||
# Prepare CIFAR10 dataset + run full noise sweep (all-in-one)
|
||
cam-benchmark-sweep-cifar10 ROWS="512" QUERIES="128":
|
||
just cam-prepare-retrieval-cifar10 {{ ROWS }} {{ QUERIES }}
|
||
just remote "python scripts/run_retrieval_noise_sweep.py --dataset outputs/cam_retrieval_benchmark/datasets/cifar10_hash512_rows{{ ROWS }}_queries{{ QUERIES }}.npz --num-rows {{ ROWS }} --output docs/cam_retrieval_noise_sweep_cifar10.md"
|
||
|
||
# Prepare CIFAR100 dataset + run full noise sweep (all-in-one)
|
||
cam-benchmark-sweep-cifar100 ROWS="512" QUERIES="128":
|
||
just cam-prepare-retrieval-cifar100 {{ ROWS }} {{ QUERIES }}
|
||
just remote "python scripts/run_retrieval_noise_sweep.py --dataset outputs/cam_retrieval_benchmark/datasets/cifar100_hash512_rows{{ ROWS }}_queries{{ QUERIES }}.npz --num-rows {{ ROWS }} --output docs/cam_retrieval_noise_sweep_cifar100.md"
|
||
|
||
# ── Remote ↔ local sync ────────────────────────────────────────────────────────
|
||
|
||
# Download benchmark outputs from remote
|
||
download-outputs:
|
||
rsync {{ rsync_flags }} {{ remote_root }}/outputs/cam_retrieval_benchmark/ outputs/cam_retrieval_benchmark/
|
||
|
||
# Download docs from remote
|
||
download-docs:
|
||
rsync {{ rsync_flags }} {{ remote_root }}/docs/ docs/
|