mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
feat(scenegraph): refactor image scene graph query into reusable function
- Export ImageSceneGraphQueryResult and query_image_against_scene_graph from scenegraph module - Replace inline hamming-distance-based image matching with dedicated query_image_against_scene_graph function - Improve top_matches structure by extracting similarity scores and hash_bytes from matches - Add .codegraph/ to gitignore (machine-local data, should not be committed) - Add CodeGraph configuration for multi-language indexing
This commit is contained in:
@@ -33,9 +33,14 @@ def base_dependencies():
|
||||
@app.cell
|
||||
def project_imports():
|
||||
"""Project module imports using new architecture."""
|
||||
from compressors import HashPipeline, hamming_distance
|
||||
from compressors import HashPipeline
|
||||
from configs import cfg_manager
|
||||
from scenegraph import ObjectNode, RoomNode, SimpleSceneGraph
|
||||
from scenegraph import (
|
||||
ObjectNode,
|
||||
RoomNode,
|
||||
SimpleSceneGraph,
|
||||
query_image_against_scene_graph,
|
||||
)
|
||||
from simulator import (
|
||||
HabitatSimulatorConfig,
|
||||
TopDownSceneElements,
|
||||
@@ -57,8 +62,8 @@ def project_imports():
|
||||
cfg_manager,
|
||||
collect_room_views_by_room,
|
||||
create_habitat_simulator,
|
||||
hamming_distance,
|
||||
numpy_to_pil,
|
||||
query_image_against_scene_graph,
|
||||
render_topdown_scene_map,
|
||||
save_object_image,
|
||||
save_room_view,
|
||||
@@ -362,13 +367,11 @@ def upload_query(mo):
|
||||
def query_matching(
|
||||
Image,
|
||||
file_upload,
|
||||
hamming_distance,
|
||||
np,
|
||||
mo,
|
||||
object_images,
|
||||
pipeline,
|
||||
query_image_against_scene_graph,
|
||||
scene_graph,
|
||||
torch,
|
||||
):
|
||||
from io import BytesIO
|
||||
|
||||
@@ -393,49 +396,34 @@ def query_matching(
|
||||
"a door",
|
||||
"a plant",
|
||||
]
|
||||
_output = pipeline.process_batch([_query_image], _text_labels, batch_size=1)
|
||||
_query_bits = (_output.hash_bits > 0).to(dtype=torch.int32)
|
||||
_query_results = query_image_against_scene_graph(
|
||||
image=_query_image,
|
||||
pipeline=pipeline,
|
||||
scene_graph=scene_graph,
|
||||
text_labels=_text_labels,
|
||||
top_k=5,
|
||||
batch_size=1,
|
||||
)
|
||||
|
||||
if _query_bits.numel() > 0 and scene_graph.objects:
|
||||
_obj_ids = list(scene_graph.objects.keys())
|
||||
_obj_hashes = []
|
||||
for _obj_id in _obj_ids:
|
||||
_obj = scene_graph.objects[_obj_id]
|
||||
_bits = np.unpackbits(np.frombuffer(_obj.visual_hash, dtype=np.uint8))[
|
||||
: pipeline.hash_bits
|
||||
].astype(np.int32)
|
||||
_obj_hashes.append(_bits)
|
||||
|
||||
_db_tensor = torch.tensor(np.array(_obj_hashes), dtype=torch.int32).to(
|
||||
_query_bits.device
|
||||
if _query_results:
|
||||
_best_result = max(
|
||||
_query_results,
|
||||
key=lambda result: result.matches[0].score if result.matches else -1,
|
||||
)
|
||||
|
||||
_distances = hamming_distance(_query_bits, _db_tensor)
|
||||
_best_query_idx = int(_distances.min(dim=1).values.argmin().item())
|
||||
|
||||
_query_tensor = _query_bits[_best_query_idx]
|
||||
query_cropped = _output.cropped_images[_best_query_idx]
|
||||
_query_distances = _distances[_best_query_idx].cpu().numpy()
|
||||
_query_hash_hex = (
|
||||
np.packbits(_query_tensor.cpu().numpy().astype(np.uint8)).tobytes().hex()
|
||||
)
|
||||
|
||||
_top_k = min(5, len(_obj_ids))
|
||||
_top_indices = np.argsort(_query_distances)[:_top_k]
|
||||
|
||||
query_cropped = _best_result.query_crop
|
||||
top_matches = [
|
||||
{
|
||||
"obj_id": _obj_ids[_i],
|
||||
"distance": int(_query_distances[_i]),
|
||||
"similarity": 1.0 - _query_distances[_i] / float(pipeline.hash_bits),
|
||||
"hash_hex": scene_graph.objects[_obj_ids[_i]].visual_hash.hex(),
|
||||
"obj_id": match.obj_id,
|
||||
"distance": int(pipeline.hash_bits - match.score),
|
||||
"similarity": match.similarity,
|
||||
"hash_hex": match.hash_bytes.hex(),
|
||||
}
|
||||
for _i in _top_indices
|
||||
for match in _best_result.matches
|
||||
]
|
||||
|
||||
query_result = {
|
||||
"query_cropped": query_cropped,
|
||||
"query_hash_hex": _query_hash_hex,
|
||||
"query_hash_hex": _best_result.query_hash.hex(),
|
||||
"top_matches": top_matches,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user