mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
feat(scenegraph): add depth-based 3D positioning via pinhole projection
- Add bbox_depth_center position strategy in SceneGraphBuilder using depth at bbox centre and configurable camera_hfov_degrees for pinhole projection. - Add optional depth_sensor_uuid to HabitatSimulatorConfig; create depth sensor spec alongside RGB sensor. - Add camera_position/camera_rotation fields to RoomView; capture pose from sensor_states when depth sensor is available. - Update flatten_room_views for backward compatibility with legacy tuple format. - Wired in depth sensor and bbox_depth_center strategy in verification notebook. - Add tests for depth sensor support and new position strategies.
This commit is contained in:
@@ -24,10 +24,9 @@ def base_dependencies():
|
||||
"""Basic dependencies for data processing."""
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
return Image, np, pl, torch
|
||||
return Image, np, pl
|
||||
|
||||
|
||||
@app.cell
|
||||
@@ -99,16 +98,15 @@ def habitat_setup(HabitatSimulatorConfig, RoomNode, create_habitat_simulator, np
|
||||
views_per_room = 12
|
||||
meters_per_pixel = 0.05
|
||||
|
||||
sim, agent = create_habitat_simulator(
|
||||
HabitatSimulatorConfig(
|
||||
scene_path=_scene_path,
|
||||
views_per_room=views_per_room,
|
||||
image_size=_image_size,
|
||||
sensor_height=1.5,
|
||||
move_forward_step=0.25,
|
||||
enable_physics=False,
|
||||
)
|
||||
habitat_config = HabitatSimulatorConfig(
|
||||
scene_path=_scene_path,
|
||||
views_per_room=views_per_room,
|
||||
image_size=_image_size,
|
||||
sensor_height=1.5,
|
||||
move_forward_step=0.25,
|
||||
enable_physics=False,
|
||||
)
|
||||
sim, agent = create_habitat_simulator(habitat_config)
|
||||
|
||||
room_nodes = []
|
||||
for idx in range(_num_rooms):
|
||||
@@ -125,7 +123,7 @@ def habitat_setup(HabitatSimulatorConfig, RoomNode, create_habitat_simulator, np
|
||||
for _node in room_nodes:
|
||||
print(f" {_node.room_id}: {_node.center}")
|
||||
|
||||
return agent, meters_per_pixel, room_nodes, sim, views_per_room
|
||||
return agent, habitat_config, meters_per_pixel, room_nodes, sim, views_per_room
|
||||
|
||||
|
||||
@app.cell
|
||||
@@ -164,6 +162,7 @@ def collect_views(
|
||||
agent,
|
||||
collect_room_views_by_room,
|
||||
flatten_room_views,
|
||||
habitat_config,
|
||||
numpy_to_pil,
|
||||
room_nodes,
|
||||
sim,
|
||||
@@ -174,6 +173,7 @@ def collect_views(
|
||||
sim=sim,
|
||||
room_nodes=room_nodes,
|
||||
views_per_room=views_per_room,
|
||||
depth_sensor_uuid=habitat_config.depth_sensor_uuid,
|
||||
)
|
||||
|
||||
room_views = flatten_room_views(all_room_views)
|
||||
@@ -193,6 +193,7 @@ def build_scene_graph(
|
||||
SceneGraphBuildConfig,
|
||||
SceneGraphBuilder,
|
||||
cfg_manager,
|
||||
habitat_config,
|
||||
mo,
|
||||
numpy_to_pil,
|
||||
pipeline,
|
||||
@@ -207,7 +208,11 @@ def build_scene_graph(
|
||||
|
||||
builder = SceneGraphBuilder(
|
||||
pipeline=pipeline,
|
||||
config=SceneGraphBuildConfig(inference_batch_size=4),
|
||||
config=SceneGraphBuildConfig(
|
||||
inference_batch_size=4,
|
||||
position_strategy="bbox_depth_center",
|
||||
camera_hfov_degrees=habitat_config.hfov_degrees,
|
||||
),
|
||||
)
|
||||
|
||||
pil_room_views = [
|
||||
@@ -218,6 +223,8 @@ def build_scene_graph(
|
||||
depth=_view.depth,
|
||||
agent_position=_view.agent_position,
|
||||
agent_rotation=_view.agent_rotation,
|
||||
camera_position=_view.camera_position,
|
||||
camera_rotation=_view.camera_rotation,
|
||||
)
|
||||
for _view in room_views
|
||||
]
|
||||
@@ -303,6 +310,8 @@ def build_tables(pl, scene_graph):
|
||||
"position_x": float(obj.position[0]) if obj.position is not None else None,
|
||||
"position_y": float(obj.position[1]) if obj.position is not None else None,
|
||||
"position_z": float(obj.position[2]) if obj.position is not None else None,
|
||||
"bbox_xyxy": str(obj.bbox_xyxy) if obj.bbox_xyxy is not None else None,
|
||||
"position_confidence": obj.position_confidence,
|
||||
}
|
||||
for obj in scene_graph.objects.values()
|
||||
]
|
||||
@@ -313,6 +322,19 @@ def build_tables(pl, scene_graph):
|
||||
return objects_df, rooms_df
|
||||
|
||||
|
||||
@app.cell
|
||||
def display_tables(mo, objects_df, rooms_df):
|
||||
mo.vstack(
|
||||
[
|
||||
mo.md("## Rooms"),
|
||||
mo.ui.table(rooms_df),
|
||||
mo.md("## Objects"),
|
||||
mo.ui.table(objects_df),
|
||||
]
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
@app.cell
|
||||
def upload_query(mo):
|
||||
file_upload = mo.ui.file(
|
||||
@@ -329,7 +351,6 @@ def query_matching(
|
||||
Image,
|
||||
file_upload,
|
||||
mo,
|
||||
object_images,
|
||||
pipeline,
|
||||
query_image_against_scene_graph,
|
||||
scene_graph,
|
||||
|
||||
Reference in New Issue
Block a user