mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
SikongJueluo
9eb52f8cef
- Evaluate multiple datasets in a single run (CIFAR10 + CIFAR100) - Report Recall@K for a list of K values from one underlying search - Save results to disk: summary.md, metrics.csv, per-dataset predictions.csv, confusion_matrix.csv - Richer evaluation output: query_labels, topk_ids, topk_labels for downstream analysis - Add --dataset and --top-k CLI overrides for benchmark command - Update config schema: dataset→datasets, top_k→top_k_list
69 lines
1.8 KiB
Python
69 lines
1.8 KiB
Python
"""Benchmark CLI command."""
|
|
|
|
import typer
|
|
from commands import app
|
|
|
|
|
|
@app.command()
|
|
def benchmark(
|
|
_ctx: typer.Context,
|
|
model_path: str | None = typer.Option(
|
|
None, "--model", "-m", help="Path to compressor model weights"
|
|
),
|
|
dataset: list[str] | None = typer.Option(
|
|
None,
|
|
"--dataset",
|
|
"-d",
|
|
help=(
|
|
"Override datasets (HuggingFace path, repeatable). "
|
|
"Each value creates a single-entry dataset list."
|
|
),
|
|
),
|
|
top_k: list[int] | None = typer.Option(
|
|
None,
|
|
"--top-k",
|
|
"-k",
|
|
help="Override top-K values (repeatable). E.g. --top-k 1 --top-k 5 --top-k 10",
|
|
),
|
|
):
|
|
"""Run benchmark evaluation across configured datasets.
|
|
|
|
Without flags, runs all datasets and K values from config.yaml.
|
|
Use --dataset to override datasets, --top-k to override K values.
|
|
"""
|
|
from benchmarks import run_benchmark
|
|
from configs import cfg_manager
|
|
from configs.models import DatasetSourceConfig
|
|
|
|
config = cfg_manager.get()
|
|
benchmark_cfg = config.benchmark
|
|
|
|
if model_path:
|
|
config.model.compressor_path = model_path
|
|
|
|
# CLI overrides
|
|
if dataset:
|
|
benchmark_cfg.datasets = [
|
|
DatasetSourceConfig(
|
|
source_type="huggingface",
|
|
path=d,
|
|
)
|
|
for d in dataset
|
|
]
|
|
|
|
if top_k:
|
|
benchmark_cfg.task.top_k_list = sorted(set(top_k))
|
|
|
|
model_name = "hash_compressor" if config.model.compressor_path else "dinov2"
|
|
output_root = config.output.directory / "benchmark"
|
|
|
|
results = run_benchmark(
|
|
model=None,
|
|
processor=None,
|
|
config=benchmark_cfg,
|
|
model_config=config.model,
|
|
model_name=model_name,
|
|
output_root=output_root,
|
|
)
|
|
return results
|