mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
feat(benchmark): support multi-dataset evaluation with configurable top-k list
- Evaluate multiple datasets in a single run (CIFAR10 + CIFAR100) - Report Recall@K for a list of K values from one underlying search - Save results to disk: summary.md, metrics.csv, per-dataset predictions.csv, confusion_matrix.csv - Richer evaluation output: query_labels, topk_ids, topk_labels for downstream analysis - Add --dataset and --top-k CLI overrides for benchmark command - Update config schema: dataset→datasets, top_k→top_k_list
This commit is contained in:
@@ -1,3 +1,5 @@
|
||||
"""Benchmark CLI command."""
|
||||
|
||||
import typer
|
||||
from commands import app
|
||||
|
||||
@@ -8,19 +10,59 @@ def benchmark(
|
||||
model_path: str | None = typer.Option(
|
||||
None, "--model", "-m", help="Path to compressor model weights"
|
||||
),
|
||||
dataset: list[str] | None = typer.Option(
|
||||
None,
|
||||
"--dataset",
|
||||
"-d",
|
||||
help=(
|
||||
"Override datasets (HuggingFace path, repeatable). "
|
||||
"Each value creates a single-entry dataset list."
|
||||
),
|
||||
),
|
||||
top_k: list[int] | None = typer.Option(
|
||||
None,
|
||||
"--top-k",
|
||||
"-k",
|
||||
help="Override top-K values (repeatable). E.g. --top-k 1 --top-k 5 --top-k 10",
|
||||
),
|
||||
):
|
||||
"""Run benchmark evaluation across configured datasets.
|
||||
|
||||
Without flags, runs all datasets and K values from config.yaml.
|
||||
Use --dataset to override datasets, --top-k to override K values.
|
||||
"""
|
||||
from benchmarks import run_benchmark
|
||||
from configs import cfg_manager
|
||||
from configs.models import DatasetSourceConfig
|
||||
|
||||
config = cfg_manager.get()
|
||||
benchmark_cfg = config.benchmark
|
||||
|
||||
if model_path:
|
||||
config.model.compressor_path = model_path
|
||||
|
||||
_ = run_benchmark(
|
||||
# CLI overrides
|
||||
if dataset:
|
||||
benchmark_cfg.datasets = [
|
||||
DatasetSourceConfig(
|
||||
source_type="huggingface",
|
||||
path=d,
|
||||
)
|
||||
for d in dataset
|
||||
]
|
||||
|
||||
if top_k:
|
||||
benchmark_cfg.task.top_k_list = sorted(set(top_k))
|
||||
|
||||
model_name = "hash_compressor" if config.model.compressor_path else "dinov2"
|
||||
output_root = config.output.directory / "benchmark"
|
||||
|
||||
results = run_benchmark(
|
||||
model=None,
|
||||
processor=None,
|
||||
config=benchmark_cfg,
|
||||
model_config=config.model,
|
||||
model_name="hash_compressor" if config.model.compressor_path else "dinov2",
|
||||
model_name=model_name,
|
||||
output_root=output_root,
|
||||
)
|
||||
return results
|
||||
|
||||
Reference in New Issue
Block a user