refactor(benchmark): delegate model loading to tasks and support CIFAR-100

- Extract model loading logic from benchmark CLI into task-owned prepare_benchmark
- Add RetrievalEncoder class wrapping DINO with optional hash compression
- Add accelerate dependency for device management  
- Switch dataset from CIFAR-10 to CIFAR-100 with fine_label column
This commit is contained in:
2026-05-09 15:12:19 +08:00
parent 0fbcd915bd
commit ab616528b4
7 changed files with 1828 additions and 1226 deletions

View File

@@ -1,17 +1,62 @@
"""Retrieval task for benchmark evaluation (Recall@K)."""
from typing import Any
from typing import TYPE_CHECKING, Any, cast
import lancedb
import pyarrow as pa
import torch
import torch.nn.functional as F
from benchmarks.base import BaseBenchmarkTask
from benchmarks.tasks.registry import RegisterTask
from compressors.model_loader import get_dino_dim, load_dino_model, load_hash_compressor
from configs import cfg_manager
from rich.progress import track
from torch import nn
from torch.utils.data import DataLoader
from transformers import BitImageProcessor
from utils.feature_extractor import extract_batch_features, infer_vector_dim
if TYPE_CHECKING:
from compressors.hash_compressor import HashCompressor
class RetrievalEncoder(nn.Module):
"""Benchmark encoder for DINO and optional hash compression."""
def __init__(
self,
dino: nn.Module,
compressor: "HashCompressor | None" = None,
) -> None:
"""Initialize retrieval encoder.
Args:
dino: DINO backbone used for feature extraction.
compressor: Optional hash compressor for recall evaluation.
"""
super().__init__()
self.dino: nn.Module = dino
self.compressor: HashCompressor | None = compressor
def forward(self, inputs: Any) -> torch.Tensor:
"""Encode processor inputs into benchmark vectors.
Args:
inputs: Batched processor outputs.
Returns:
Float tensor used for LanceDB insertion and retrieval.
"""
outputs = self.dino(**inputs)
tokens = outputs.last_hidden_state
if self.compressor is None:
features = tokens.mean(dim=1)
return F.normalize(features, dim=-1)
bits = self.compressor.encode(tokens)
return bits.to(dtype=torch.float32)
def _build_eval_schema(vector_dim: int) -> pa.Schema:
"""Build PyArrow schema for evaluation database table.
@@ -35,7 +80,7 @@ def _establish_eval_database(
processor: BitImageProcessor,
model: nn.Module,
table: lancedb.table.Table,
dataloader: DataLoader,
dataloader: DataLoader[Any],
) -> None:
"""Extract features from training images and store them in a database table.
@@ -47,11 +92,12 @@ def _establish_eval_database(
"""
# Extract all features using the utility function
all_features = extract_batch_features(processor, model, dataloader)
config = cfg_manager.get()
# Store features to database
global_idx = 0
for batch in track(dataloader, description="Storing eval database"):
labels = batch["label"]
labels = batch[config.benchmark.dataset.label_column]
labels_list = labels.tolist()
batch_size = len(labels_list)
@@ -72,7 +118,7 @@ def _evaluate_recall(
processor: BitImageProcessor,
model: nn.Module,
table: lancedb.table.Table,
dataloader: DataLoader,
dataloader: DataLoader[Any],
top_k: int,
) -> tuple[int, int]:
"""Evaluate Recall@K by searching the database for each test image.
@@ -89,13 +135,14 @@ def _evaluate_recall(
"""
# Extract all features using the utility function
all_features = extract_batch_features(processor, model, dataloader)
config = cfg_manager.get()
correct = 0
total = 0
feature_idx = 0
for batch in track(dataloader, description=f"Evaluating Recall@{top_k}"):
labels = batch["label"]
labels = batch[config.benchmark.dataset.label_column]
labels_list = labels.tolist()
for j in range(len(labels_list)):
@@ -123,14 +170,79 @@ def _evaluate_recall(
class RetrievalTask(BaseBenchmarkTask):
"""Retrieval evaluation task (Recall@K)."""
def __init__(self, top_k: int = 10):
def __init__(
self,
top_k: int = 10,
dino_model: str = "facebook/dinov2-large",
compression_dim: int = 512,
compressor_path: str | None = None,
):
"""Initialize retrieval task.
Args:
top_k: Number of top results to retrieve for recall calculation.
dino_model: DINO model name used for feature extraction.
compression_dim: Output dimension of the hash compressor.
compressor_path: Optional path to trained hash compressor weights.
"""
super().__init__(top_k=top_k)
self.top_k = top_k
self.dino_model = dino_model
self.compression_dim = compression_dim
self.compressor_path = compressor_path
self._processor: BitImageProcessor | None = None
self._model: nn.Module | None = None
self._model_name = "hash_compressor" if compressor_path else "dinov2"
def prepare_benchmark(
self,
model: Any,
processor: Any,
model_name: str = "model",
) -> tuple[nn.Module, BitImageProcessor, str]:
"""Resolve benchmark resources for this task.
Args:
model: Optional pre-built model from the caller.
processor: Optional pre-built processor from the caller.
model_name: Fallback table model name.
Returns:
Tuple of benchmark model, processor, and resolved model name.
"""
if model is not None and processor is not None:
return (
cast(nn.Module, model),
cast(BitImageProcessor, processor),
model_name,
)
self._ensure_resources_loaded()
return (
cast(nn.Module, self._model),
cast(BitImageProcessor, self._processor),
self._model_name,
)
def _ensure_resources_loaded(self) -> None:
"""Lazy-load retrieval benchmark resources."""
if self._processor is not None and self._model is not None:
return
processor, dino = load_dino_model(self.dino_model)
compressor = None
if self.compressor_path is not None:
compressor = load_hash_compressor(
input_dim=get_dino_dim(self.dino_model),
hash_bits=self.compression_dim,
compressor_path=self.compressor_path,
)
compressor.eval()
self._processor = processor
self._model = RetrievalEncoder(dino=dino, compressor=compressor)
self._model.eval()
def build_database(
self,