SikongJueluo/Mini-Nav
1
0
Fork 0
mirror of https://github.com/SikongJueluo/Mini-Nav.git synced 2026-07-12 20:15:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(scenegraph): add SceneGraphBuilder for pipeline-driven graph construction

Browse Source 
Introduce SceneGraphBuilder + SceneGraphBuildConfig to decouple scene graph
construction from the verification notebook. The builder handles batch
inference, hash encoding, and object node creation internally.

- Add SceneGraphBuilder.build_from_room_views() as the main entry point
- Add SceneGraphBuildConfig for inference_batch_size and position strategy
- Add SceneGraphBuildArtifacts to carry cropped images and debug metadata
- Extend ObjectNode with optional detection metadata (label, confidence,
  bbox_xyxy, source_view_id, position_confidence)
- Add RoomView frozen dataclass as a structured view container
- Add flatten_room_views() utility to replace inline list comprehensions
- Refactor notebooks/verification.py to use the new builder API

BREAKING CHANGE: ObjectNode now accepts additional optional fields; direct
scene_graph.objects[obj_id] = ObjectNode(...) construction in the notebook
is replaced by builder.build_from_room_views(...).
This commit is contained in:
SikongJueluo
2026-05-30 15:40:58 +08:00
parent 97e53d44f8
commit a127032e18
8 changed files with 910 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
notebooks/verification.py
View File

@@ -36,10 +36,9 @@ def project_imports():
from compressors import HashPipeline
from configs import cfg_manager
from scenegraph import (
ObjectNode,
RoomNode,
SimpleSceneGraph,
bits_tensor_to_hash_bytes,
SceneGraphBuildConfig,
SceneGraphBuilder,
query_image_against_scene_graph,
)
from simulator import (
@@ -47,6 +46,7 @@ def project_imports():
TopDownSceneElements,
collect_room_views_by_room,
create_habitat_simulator,
flatten_room_views,
render_topdown_scene_map,
save_object_image,
save_room_view,
@@ -56,14 +56,14 @@ def project_imports():
return (
HashPipeline,
HabitatSimulatorConfig,
ObjectNode,
RoomNode,
SimpleSceneGraph,
SceneGraphBuildConfig,
SceneGraphBuilder,
TopDownSceneElements,
bits_tensor_to_hash_bytes,
cfg_manager,
collect_room_views_by_room,
create_habitat_simulator,
flatten_room_views,
numpy_to_pil,
query_image_against_scene_graph,
render_topdown_scene_map,
@@ -163,6 +163,7 @@ def pipeline_init(HashPipeline):
def collect_views(
agent,
collect_room_views_by_room,
flatten_room_views,
numpy_to_pil,
room_nodes,
sim,
@@ -175,162 +176,108 @@ def collect_views(
views_per_room=views_per_room,
)
# Flatten room views into (room_id, view_idx, PIL.Image) tuples.
room_views = flatten_room_views(all_room_views)
# Build dataset of (room_id, view_idx, PIL.Image) tuples.
room_view_dataset = [
(_room_id, _view_idx, numpy_to_pil(_rgb))
for _room_id, _views in all_room_views.items()
for _view_idx, _rgb in enumerate(_views)
(_view.room_id, _view.view_idx, numpy_to_pil(_view.rgb))
for _view in room_views
]
print(f"Collected {len(room_view_dataset)} room views")
return all_room_views, room_view_dataset
return all_room_views, room_view_dataset, room_views
@app.cell
def build_scene_graph(
ObjectNode,
SimpleSceneGraph,
bits_tensor_to_hash_bytes,
SceneGraphBuildConfig,
SceneGraphBuilder,
cfg_manager,
mo,
np,
numpy_to_pil,
pipeline,
room_nodes,
room_view_dataset,
room_views,
save_object_image,
save_room_view,
text_labels,
torch,
):
scene_graph = SimpleSceneGraph(
rooms={_room.room_id: _room for _room in room_nodes},
objects={},
)
# Storage for cropped object images (for visualization).
object_images = {}
output_dir = cfg_manager.get().output.directory / "verification"
output_dir.mkdir(parents=True, exist_ok=True)
_images = [item[2] for item in room_view_dataset]
_metadata = [(item[0], item[1]) for item in room_view_dataset]
builder = SceneGraphBuilder(
pipeline=pipeline,
config=SceneGraphBuildConfig(inference_batch_size=4),
)
inference_batch_size = 4
image_batches = [
_images[index : index + inference_batch_size]
for index in range(0, len(_images), inference_batch_size)
pil_room_views = [
type(_view)(
room_id=_view.room_id,
view_idx=_view.view_idx,
rgb=numpy_to_pil(_view.rgb),
depth=_view.depth,
agent_position=_view.agent_position,
agent_rotation=_view.agent_rotation,
)
for _view in room_views
]
_cropped_images = []
debug_meta = []
hash_batches = []
for _batch_images in mo.status.progress_bar(
image_batches,
title="Running pipeline inference on room views",
subtitle=f"Batch size {inference_batch_size} with ETA",
completion_title="Pipeline inference finished",
completion_subtitle=(
f"Processed {len(_images)} room views in {len(image_batches)} batches"
),
with mo.status.spinner(title="Building scene graph from room views"):
scene_graph, build_artifacts = builder.build_from_room_views(
room_nodes=room_nodes,
room_views=pil_room_views,
text_labels=text_labels,
)
object_images = build_artifacts.object_images
debug_meta = build_artifacts.debug_meta
# Save original room views.
for _room_view in mo.status.progress_bar(
pil_room_views,
title="Saving room-view snapshots",
subtitle="Writing original room images to disk",
completion_title="Room-view snapshots saved",
completion_subtitle=f"Saved {len(pil_room_views)} room views",
show_eta=True,
show_rate=True,
remove_on_exit=False,
):
_batch_output = pipeline.process_batch(
_batch_images,
text_labels,
batch_size=inference_batch_size,
return_debug_details=True,
)
_cropped_images.extend(_batch_output.cropped_images)
debug_meta.extend(_batch_output.debug_meta)
if _batch_output.hash_bits.numel() > 0:
hash_batches.append(_batch_output.hash_bits)
save_room_view(output_dir, _room_view.room_id, _room_view.view_idx, _room_view.rgb)
if hash_batches:
hash_tensor = torch.cat(hash_batches, dim=0)
else:
hash_tensor = torch.empty(
(0, pipeline.hash_bits), dtype=torch.int32, device=pipeline.device
# Save object crops.
for _obj_id, _cropped in mo.status.progress_bar(
object_images.items(),
title="Saving object crops",
subtitle="Writing cropped object images to disk",
completion_title="Object crops saved",
completion_subtitle=f"Saved {len(object_images)} object crops",
show_eta=True,
show_rate=True,
remove_on_exit=False,
):
_node = scene_graph.objects[_obj_id]
save_object_image(
output_dir,
_node.room_id,
_obj_id,
_node.last_seen_frame,
0, # mask idx 0 ok for M0
_cropped,
)
from collections import Counter
_reasons = Counter(m["fallback_reason"] or "ok" for m in debug_meta)
_meta_fallbacks = [_meta.get("fallback_reason") for _meta in debug_meta]
fallback_count = sum(1 for f in _meta_fallbacks if f is not None)
_reasons = Counter(f or "ok" for f in _meta_fallbacks)
print(f"Fallback breakdown: {_reasons}")
# Save original room views.
for _room_id, _view_idx, _image in mo.status.progress_bar(
room_view_dataset,
title="Saving room-view snapshots",
subtitle="Writing original room images to disk",
completion_title="Room-view snapshots saved",
completion_subtitle=f"Saved {len(room_view_dataset)} room views",
show_eta=True,
show_rate=True,
remove_on_exit=False,
):
save_room_view(output_dir, _room_id, _view_idx, _image)
# Prefix sum: map flat crop index to (input_image_idx, mask_idx).
_num_selected = [_m["num_selected"] for _m in debug_meta]
assert sum(_num_selected) == len(_cropped_images), (
f"Sum of num_selected ({sum(_num_selected)}) != cropped_images count ({len(_cropped_images)})"
)
_prefix_sums = [0]
for _n in _num_selected:
_prefix_sums.append(_prefix_sums[-1] + _n)
_obj_counter = 0
_total_crops = len(_cropped_images)
object_tasks = []
for _img_idx, _n_crops in enumerate(_num_selected):
_room_id, _view_idx = _metadata[_img_idx]
for _mask_idx in range(_n_crops):
object_tasks.append((_img_idx, _room_id, _view_idx, _mask_idx, _n_crops))
for _img_idx, _room_id, _view_idx, _mask_idx, _n_crops in mo.status.progress_bar(
object_tasks,
title="Building scene graph objects",
subtitle="Preparing cropped objects and hashes with ETA",
completion_title="Scene graph build complete",
completion_subtitle=f"Created {_total_crops} cropped object entries",
show_eta=True,
show_rate=True,
remove_on_exit=False,
):
_crop_flat_idx = _prefix_sums[_img_idx] + _mask_idx
_cropped = _cropped_images[_crop_flat_idx]
_hash_bits = hash_tensor[_crop_flat_idx]
_obj_id = f"{_room_id}_v{_view_idx:03d}_m{_mask_idx:02d}"
_hash_bytes = bits_tensor_to_hash_bytes(_hash_bits)
object_images[_obj_id] = _cropped
save_object_image(output_dir, _room_id, _obj_id, _view_idx, _mask_idx, _cropped)
scene_graph.objects[_obj_id] = ObjectNode(
obj_id=_obj_id,
room_id=_room_id,
position=np.array([0.0, 0.0, 0.0], dtype=np.float32),
visual_hash=_hash_bytes,
semantic_hash=_hash_bytes,
hit_count=1,
last_seen_frame=_view_idx,
)
_obj_counter += 1
_fallback_count = sum(
1 for _meta in debug_meta if _meta["fallback_reason"] is not None
)
print(f"Created {_obj_counter} objects")
print(f"Created {len(scene_graph.objects)} objects")
print(f"Saved cropped images to: {output_dir}")
print(f"Fallback frames: {_fallback_count}/{len(debug_meta)}")
print(f"Fallback frames: {fallback_count}/{len(debug_meta)}")
return hash_tensor, object_images, output_dir, scene_graph
return build_artifacts, object_images, output_dir, scene_graph
@app.cell
@@ -350,6 +297,12 @@ def build_tables(pl, scene_graph):
"obj_id": obj.obj_id,
"room_id": obj.room_id,
"visual_hash": obj.visual_hash.hex()[:16] + "...",
"label": obj.label,
"confidence": obj.confidence,
"source_view_id": obj.source_view_id,
"position_x": float(obj.position[0]) if obj.position is not None else None,
"position_y": float(obj.position[1]) if obj.position is not None else None,
"position_z": float(obj.position[2]) if obj.position is not None else None,
}
for obj in scene_graph.objects.values()
]