feat(scenegraph): add depth-based 3D positioning via pinhole projection

- Add bbox_depth_center position strategy in SceneGraphBuilder using depth
  at bbox centre and configurable camera_hfov_degrees for pinhole projection.
- Add optional depth_sensor_uuid to HabitatSimulatorConfig; create depth
  sensor spec alongside RGB sensor.
- Add camera_position/camera_rotation fields to RoomView; capture pose from
  sensor_states when depth sensor is available.
- Update flatten_room_views for backward compatibility with legacy tuple
  format.
- Wired in depth sensor and bbox_depth_center strategy in verification
  notebook.
- Add tests for depth sensor support and new position strategies.
This commit is contained in:
2026-05-31 14:29:05 +08:00
parent a127032e18
commit 7a1e1ccf3f
7 changed files with 1729 additions and 38 deletions

View File

@@ -24,10 +24,9 @@ def base_dependencies():
"""Basic dependencies for data processing."""
import numpy as np
import polars as pl
import torch
from PIL import Image
return Image, np, pl, torch
return Image, np, pl
@app.cell
@@ -99,16 +98,15 @@ def habitat_setup(HabitatSimulatorConfig, RoomNode, create_habitat_simulator, np
views_per_room = 12
meters_per_pixel = 0.05
sim, agent = create_habitat_simulator(
HabitatSimulatorConfig(
scene_path=_scene_path,
views_per_room=views_per_room,
image_size=_image_size,
sensor_height=1.5,
move_forward_step=0.25,
enable_physics=False,
)
habitat_config = HabitatSimulatorConfig(
scene_path=_scene_path,
views_per_room=views_per_room,
image_size=_image_size,
sensor_height=1.5,
move_forward_step=0.25,
enable_physics=False,
)
sim, agent = create_habitat_simulator(habitat_config)
room_nodes = []
for idx in range(_num_rooms):
@@ -125,7 +123,7 @@ def habitat_setup(HabitatSimulatorConfig, RoomNode, create_habitat_simulator, np
for _node in room_nodes:
print(f" {_node.room_id}: {_node.center}")
return agent, meters_per_pixel, room_nodes, sim, views_per_room
return agent, habitat_config, meters_per_pixel, room_nodes, sim, views_per_room
@app.cell
@@ -164,6 +162,7 @@ def collect_views(
agent,
collect_room_views_by_room,
flatten_room_views,
habitat_config,
numpy_to_pil,
room_nodes,
sim,
@@ -174,6 +173,7 @@ def collect_views(
sim=sim,
room_nodes=room_nodes,
views_per_room=views_per_room,
depth_sensor_uuid=habitat_config.depth_sensor_uuid,
)
room_views = flatten_room_views(all_room_views)
@@ -193,6 +193,7 @@ def build_scene_graph(
SceneGraphBuildConfig,
SceneGraphBuilder,
cfg_manager,
habitat_config,
mo,
numpy_to_pil,
pipeline,
@@ -207,7 +208,11 @@ def build_scene_graph(
builder = SceneGraphBuilder(
pipeline=pipeline,
config=SceneGraphBuildConfig(inference_batch_size=4),
config=SceneGraphBuildConfig(
inference_batch_size=4,
position_strategy="bbox_depth_center",
camera_hfov_degrees=habitat_config.hfov_degrees,
),
)
pil_room_views = [
@@ -218,6 +223,8 @@ def build_scene_graph(
depth=_view.depth,
agent_position=_view.agent_position,
agent_rotation=_view.agent_rotation,
camera_position=_view.camera_position,
camera_rotation=_view.camera_rotation,
)
for _view in room_views
]
@@ -303,6 +310,8 @@ def build_tables(pl, scene_graph):
"position_x": float(obj.position[0]) if obj.position is not None else None,
"position_y": float(obj.position[1]) if obj.position is not None else None,
"position_z": float(obj.position[2]) if obj.position is not None else None,
"bbox_xyxy": str(obj.bbox_xyxy) if obj.bbox_xyxy is not None else None,
"position_confidence": obj.position_confidence,
}
for obj in scene_graph.objects.values()
]
@@ -313,6 +322,19 @@ def build_tables(pl, scene_graph):
return objects_df, rooms_df
@app.cell
def display_tables(mo, objects_df, rooms_df):
mo.vstack(
[
mo.md("## Rooms"),
mo.ui.table(rooms_df),
mo.md("## Objects"),
mo.ui.table(objects_df),
]
)
return
@app.cell
def upload_query(mo):
file_upload = mo.ui.file(
@@ -329,7 +351,6 @@ def query_matching(
Image,
file_upload,
mo,
object_images,
pipeline,
query_image_against_scene_graph,
scene_graph,