mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
SikongJueluo
7a1e1ccf3f
- Add bbox_depth_center position strategy in SceneGraphBuilder using depth at bbox centre and configurable camera_hfov_degrees for pinhole projection. - Add optional depth_sensor_uuid to HabitatSimulatorConfig; create depth sensor spec alongside RGB sensor. - Add camera_position/camera_rotation fields to RoomView; capture pose from sensor_states when depth sensor is available. - Update flatten_room_views for backward compatibility with legacy tuple format. - Wired in depth sensor and bbox_depth_center strategy in verification notebook. - Add tests for depth sensor support and new position strategies.
1574 lines
51 KiB
Python
1574 lines
51 KiB
Python
"""Tests for scenegraph builder / ObjectNode."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import sys
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
import numpy as np
|
||
import pytest
|
||
import torch
|
||
|
||
MINI_NAV_DIR = Path(__file__).resolve().parents[1] / "mini-nav"
|
||
sys.path.insert(0, str(MINI_NAV_DIR))
|
||
|
||
from scenegraph.hash_codec import bits_tensor_to_hash_bytes # noqa: E402
|
||
from scenegraph.objectnode import ObjectNode # noqa: E402
|
||
|
||
|
||
def _hash_bytes() -> bytes:
|
||
"""Create a 512-bit hash with bit 0 set to 1."""
|
||
bits = torch.zeros(512, dtype=torch.uint8)
|
||
bits[0] = 1
|
||
return bits_tensor_to_hash_bytes(bits)
|
||
|
||
|
||
def test_object_node_accepts_optional_detection_metadata():
|
||
"""ObjectNode should accept and preserve optional detection metadata."""
|
||
hb = _hash_bytes()
|
||
node = ObjectNode(
|
||
obj_id="room_a_v000_m00",
|
||
room_id="room_a",
|
||
position=np.array([1, 2, 3], dtype=np.float32),
|
||
visual_hash=hb,
|
||
semantic_hash=hb,
|
||
hit_count=1,
|
||
last_seen_frame=0,
|
||
label="a chair",
|
||
confidence=0.87,
|
||
bbox_xyxy=(10, 20, 30, 40),
|
||
source_view_id="room_a_v000",
|
||
position_confidence=0.0,
|
||
)
|
||
|
||
assert node.obj_id == "room_a_v000_m00"
|
||
assert node.room_id == "room_a"
|
||
np.testing.assert_array_equal(node.position, np.array([1, 2, 3], dtype=np.float32))
|
||
assert node.visual_hash == hb
|
||
assert node.semantic_hash == hb
|
||
assert node.hit_count == 1
|
||
assert node.last_seen_frame == 0
|
||
assert node.label == "a chair"
|
||
assert node.confidence == 0.87
|
||
assert node.bbox_xyxy == (10, 20, 30, 40)
|
||
assert node.source_view_id == "room_a_v000"
|
||
assert node.position_confidence == 0.0
|
||
|
||
|
||
def test_room_view_equality_with_numpy_rgb_does_not_compare_payload():
|
||
"""RoomView equality should not attempt element-wise numpy comparison."""
|
||
from simulator import RoomView # noqa: E402
|
||
|
||
rgb_a = np.zeros((2, 2, 3), dtype=np.uint8)
|
||
rgb_b = np.ones((2, 2, 3), dtype=np.uint8)
|
||
|
||
# Two views with different numpy payloads — still equal because
|
||
# rgb is excluded from equality (only room_id, view_idx matter).
|
||
v1 = RoomView(room_id="room_a", view_idx=0, rgb=rgb_a)
|
||
v2 = RoomView(room_id="room_a", view_idx=0, rgb=rgb_b)
|
||
assert v1 == v2
|
||
|
||
# Different room_id → not equal
|
||
v3 = RoomView(room_id="room_b", view_idx=0, rgb=rgb_a)
|
||
assert v1 != v3
|
||
|
||
# Different view_idx → not equal
|
||
v4 = RoomView(room_id="room_a", view_idx=1, rgb=rgb_a)
|
||
assert v1 != v4
|
||
|
||
|
||
def test_flatten_room_views_with_numpy_arrays_safe_equality():
|
||
"""flatten_room_views with numpy arrays should not raise ambiguous truth value errors."""
|
||
from simulator import RoomView, flatten_room_views # noqa: E402
|
||
|
||
rgb_a = np.zeros((2, 2, 3), dtype=np.uint8)
|
||
rgb_b = np.ones((2, 2, 3), dtype=np.uint8)
|
||
|
||
views = flatten_room_views({"room_a": [rgb_a, rgb_b]})
|
||
|
||
# Compare with expected list — uses identity-field comparison only
|
||
expected = [
|
||
RoomView(room_id="room_a", view_idx=0, rgb=rgb_a),
|
||
RoomView(room_id="room_a", view_idx=1, rgb=rgb_b),
|
||
]
|
||
assert views == expected
|
||
|
||
# Even with wildly different numpy arrays, comparison is safe
|
||
swapped = [
|
||
RoomView(room_id="room_a", view_idx=0, rgb=rgb_b),
|
||
RoomView(room_id="room_a", view_idx=1, rgb=rgb_a),
|
||
]
|
||
assert views == swapped # rgb payload ignored
|
||
|
||
# Mismatched identity fields still differ
|
||
different_room = [
|
||
RoomView(room_id="room_b", view_idx=0, rgb=rgb_a),
|
||
RoomView(room_id="room_a", view_idx=1, rgb=rgb_b),
|
||
]
|
||
assert views != different_room
|
||
|
||
|
||
def test_flatten_room_views_preserves_room_and_view_identity():
|
||
from PIL import Image # noqa: E402
|
||
from simulator import RoomView, flatten_room_views # noqa: E402
|
||
|
||
first = Image.new("RGB", (4, 4), "red")
|
||
second = Image.new("RGB", (4, 4), "blue")
|
||
views = flatten_room_views({"room_a": [first, second]})
|
||
|
||
# Ensure the returned objects are the same instances
|
||
assert views[0].rgb is first
|
||
assert views[1].rgb is second
|
||
|
||
# Equality compares only room_id and view_idx (payload excluded).
|
||
# Different PIL images with same room_id/view_idx compare equal.
|
||
assert views == [
|
||
RoomView(room_id="room_a", view_idx=0, rgb=first),
|
||
RoomView(room_id="room_a", view_idx=1, rgb=second),
|
||
]
|
||
assert views == [
|
||
RoomView(room_id="room_a", view_idx=0, rgb=second), # rgb ignored
|
||
RoomView(room_id="room_a", view_idx=1, rgb=first), # rgb ignored
|
||
]
|
||
|
||
# Different room_id or view_idx makes them unequal
|
||
assert views != [
|
||
RoomView(room_id="room_b", view_idx=0, rgb=first),
|
||
RoomView(room_id="room_a", view_idx=1, rgb=second),
|
||
]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# SceneGraphBuilder M0 tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class FakePipeline:
|
||
"""Fake pipeline that records calls and returns a pre-set output."""
|
||
|
||
def __init__(self, output, calls, hash_bits=512):
|
||
self.output = output
|
||
self.calls = calls
|
||
self._hash_bits = hash_bits
|
||
|
||
def process_batch(self, images, text_labels, batch_size=32, return_debug_details=False):
|
||
self.calls.append({
|
||
"images": images,
|
||
"text_labels": text_labels,
|
||
"batch_size": batch_size,
|
||
"return_debug_details": return_debug_details,
|
||
})
|
||
return self.output
|
||
|
||
|
||
def test_scene_graph_builder_creates_objects_from_pipeline_debug_meta():
|
||
"""Build a graph from one room view with one detection and verify all fields."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
rgb = np.zeros((32, 32, 3), dtype=np.uint8)
|
||
room_view = RoomView(room_id="room_a", view_idx=0, rgb=rgb)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
|
||
crop = "crop_data_0"
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[10.0, 20.0, 30.0, 40.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((32, 32), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[], hash_bits=512)
|
||
config = SceneGraphBuildConfig(inference_batch_size=7)
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
# Pipeline was called correctly
|
||
assert len(pipeline.calls) == 1
|
||
call = pipeline.calls[0]
|
||
assert call["return_debug_details"] is True
|
||
assert call["batch_size"] == 7
|
||
assert len(call["images"]) == 1
|
||
assert call["images"][0] is rgb
|
||
assert call["text_labels"] == text_labels
|
||
|
||
# Graph rooms
|
||
assert list(graph.rooms.keys()) == ["room_a"]
|
||
assert graph.rooms["room_a"] is room_node
|
||
|
||
# Graph objects
|
||
assert list(graph.objects.keys()) == ["room_a_v000_m00"]
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
assert obj.room_id == "room_a"
|
||
np.testing.assert_array_equal(obj.position, np.array([10, 0, 20], dtype=np.float32))
|
||
|
||
expected_hash = bits_tensor_to_hash_bytes(hash_bits[0])
|
||
assert obj.visual_hash == expected_hash
|
||
assert obj.semantic_hash == expected_hash
|
||
|
||
assert obj.label == "a chair"
|
||
assert obj.confidence == 0.91
|
||
assert obj.bbox_xyxy == (10.0, 20.0, 30.0, 40.0)
|
||
assert obj.source_view_id == "room_a_v000"
|
||
assert obj.position_confidence == 0.0
|
||
|
||
# Artifacts
|
||
assert artifacts.object_images["room_a_v000_m00"] is crop
|
||
assert artifacts.debug_meta == [debug_meta_entry]
|
||
|
||
|
||
def test_scene_graph_builder_handles_empty_pipeline_output():
|
||
"""Pipeline returning no detections should produce rooms-only graph."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[0, 0, 0], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
rgb = np.zeros((32, 32, 3), dtype=np.uint8)
|
||
room_view = RoomView(room_id="room_a", view_idx=0, rgb=rgb)
|
||
room_views = [room_view]
|
||
|
||
text_labels: list[str] = []
|
||
|
||
hash_bits = torch.zeros(0, 512, dtype=torch.uint8)
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [],
|
||
"boxes_xyxy": [],
|
||
"scores": [],
|
||
"labels": [],
|
||
"masks": [],
|
||
"num_selected": 0,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
builder = SceneGraphBuilder(pipeline=pipeline)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
# Only rooms, no objects
|
||
assert list(graph.rooms.keys()) == ["room_a"]
|
||
assert graph.objects == {}
|
||
assert artifacts.object_images == {}
|
||
|
||
|
||
def test_config_rejects_enable_fusion():
|
||
"""SceneGraphBuildConfig should reject enable_fusion=True."""
|
||
from scenegraph import SceneGraphBuildConfig
|
||
|
||
with pytest.raises(ValueError, match="fusion"):
|
||
SceneGraphBuildConfig(enable_fusion=True)
|
||
|
||
|
||
def test_config_rejects_non_room_center_strategy():
|
||
"""SceneGraphBuildConfig should reject non-room_center position_strategy."""
|
||
from scenegraph import SceneGraphBuildConfig
|
||
|
||
with pytest.raises(ValueError, match="room_center"):
|
||
SceneGraphBuildConfig(position_strategy="global")
|
||
|
||
|
||
def test_config_rejects_zero_batch_size():
|
||
"""SceneGraphBuildConfig should reject inference_batch_size <= 0."""
|
||
from scenegraph import SceneGraphBuildConfig
|
||
|
||
with pytest.raises(ValueError):
|
||
SceneGraphBuildConfig(inference_batch_size=0)
|
||
|
||
|
||
def test_scene_graph_builder_public_method_stays_small():
|
||
"""build_from_room_views should stay an orchestration method, not own all details."""
|
||
import inspect
|
||
|
||
from scenegraph import SceneGraphBuilder
|
||
|
||
source = inspect.getsource(SceneGraphBuilder.build_from_room_views)
|
||
code_lines = [
|
||
line for line in source.splitlines()
|
||
if line.strip() and not line.strip().startswith("#")
|
||
]
|
||
|
||
assert len(code_lines) <= 45
|
||
for helper_name in [
|
||
"_prepare_graph",
|
||
"_run_pipeline",
|
||
"_build_artifacts",
|
||
"_validate_pipeline_output",
|
||
"_add_objects_to_graph",
|
||
]:
|
||
assert helper_name in source
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Validation error tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_builder_raises_on_selected_indices_num_selected_mismatch():
|
||
"""selected_indices length != num_selected should raise ValueError."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[0, 0, 0], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
rgb = np.zeros((32, 32, 3), dtype=np.uint8)
|
||
room_view = RoomView(room_id="room_a", view_idx=0, rgb=rgb)
|
||
room_views = [room_view]
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0], # len 1
|
||
"boxes_xyxy": [[10.0, 20.0, 30.0, 40.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"num_selected": 0, # mismatch!
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=["crop"],
|
||
hash_bits=torch.zeros(1, 512, dtype=torch.uint8),
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
builder = SceneGraphBuilder(pipeline=pipeline)
|
||
|
||
with pytest.raises(ValueError, match="selected_indices"):
|
||
builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=[],
|
||
)
|
||
|
||
|
||
def test_builder_raises_on_selected_idx_out_of_labels_range():
|
||
"""selected_idx >= len(labels) should raise ValueError."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[0, 0, 0], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
rgb = np.zeros((32, 32, 3), dtype=np.uint8)
|
||
room_view = RoomView(room_id="room_a", view_idx=0, rgb=rgb)
|
||
room_views = [room_view]
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [5], # index 5, but labels has only 1 entry
|
||
"boxes_xyxy": [[10.0, 20.0, 30.0, 40.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=["crop"],
|
||
hash_bits=torch.zeros(1, 512, dtype=torch.uint8),
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
builder = SceneGraphBuilder(pipeline=pipeline)
|
||
|
||
with pytest.raises(ValueError, match="selected_idx.*labels"):
|
||
builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=[],
|
||
)
|
||
|
||
|
||
def test_builder_raises_on_missing_room_before_pipeline():
|
||
"""Unknown room_id in room_view should raise ValueError before pipeline call."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[0, 0, 0], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
rgb = np.zeros((32, 32, 3), dtype=np.uint8)
|
||
room_view = RoomView(room_id="room_b", view_idx=0, rgb=rgb) # not in room_nodes!
|
||
room_views = [room_view]
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[],
|
||
hash_bits=torch.zeros(0, 512, dtype=torch.uint8),
|
||
debug_meta=[],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
builder = SceneGraphBuilder(pipeline=pipeline)
|
||
|
||
with pytest.raises(ValueError, match="room_b"):
|
||
builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=[],
|
||
)
|
||
|
||
# Pipeline should NOT have been called — error raised before pipeline
|
||
assert len(pipeline.calls) == 0
|
||
|
||
|
||
def test_builder_raises_on_debug_meta_length_mismatch():
|
||
"""Mismatched debug_meta length should raise ValueError."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[0, 0, 0], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
rgb = np.zeros((32, 32, 3), dtype=np.uint8)
|
||
room_view = RoomView(room_id="room_a", view_idx=0, rgb=rgb)
|
||
room_views = [room_view]
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [],
|
||
"boxes_xyxy": [],
|
||
"scores": [],
|
||
"labels": [],
|
||
"num_selected": 0,
|
||
}
|
||
|
||
# 2 debug_meta entries but only 1 room_view
|
||
output = SimpleNamespace(
|
||
cropped_images=[],
|
||
hash_bits=torch.zeros(0, 512, dtype=torch.uint8),
|
||
debug_meta=[debug_meta_entry, debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
builder = SceneGraphBuilder(pipeline=pipeline)
|
||
|
||
with pytest.raises(ValueError, match="debug_meta"):
|
||
builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=[],
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Scene graph depth positioning M1 — depth + pose capture in RoomView
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class FakeAgentState:
|
||
"""Fake agent state with position, rotation, and optional sensor_states."""
|
||
|
||
def __init__(self, position=None, rotation=None, sensor_states=None):
|
||
self.position = position if position is not None else np.array([0.0, 0.0, 0.0], dtype=np.float32)
|
||
self.rotation = rotation if rotation is not None else np.eye(3, dtype=np.float32)
|
||
self.sensor_states = sensor_states
|
||
|
||
|
||
class FakeHabitatModuleForViews:
|
||
"""Fake habitat_sim module exposing AgentState."""
|
||
|
||
AgentState = FakeAgentState
|
||
|
||
|
||
class FakeAgentForViews:
|
||
"""Fake agent that tracks set_state and returns copied state from get_state."""
|
||
|
||
def __init__(self):
|
||
self._state = FakeAgentState()
|
||
self.set_state_calls = []
|
||
|
||
def set_state(self, state):
|
||
# Preserve sensor_states from the old state when the new state does not
|
||
# have any. This mimics real habitat behaviour: creating a bare
|
||
# AgentState() and calling set_state() does not destroy the sim's
|
||
# sensor configuration.
|
||
if state.sensor_states is None and hasattr(self, "_state"):
|
||
old_ss = getattr(self._state, "sensor_states", None)
|
||
if old_ss is not None:
|
||
state.sensor_states = old_ss
|
||
self._state = state
|
||
self.set_state_calls.append(state)
|
||
|
||
def get_state(self):
|
||
import copy
|
||
return copy.copy(self._state)
|
||
|
||
|
||
class FakeSimForViews:
|
||
"""Fake simulator returning observation dicts and recording steps."""
|
||
|
||
def __init__(self, observations_by_step=None):
|
||
self.observations_by_step = observations_by_step or {}
|
||
self.steps = []
|
||
|
||
def get_sensor_observations(self):
|
||
step_key = len(self.steps)
|
||
if step_key in self.observations_by_step:
|
||
return self.observations_by_step[step_key]
|
||
return {"color_sensor": np.zeros((4, 4, 3), dtype=np.uint8),
|
||
"depth_sensor": np.zeros((4, 4), dtype=np.float32)}
|
||
|
||
def step(self, action):
|
||
self.steps.append(action)
|
||
|
||
|
||
def test_collect_room_views_by_room_populates_depth_and_pose():
|
||
"""collect_room_views_by_room with depth_sensor_uuid should populate RoomView depth/pose fields."""
|
||
from types import SimpleNamespace
|
||
|
||
from scenegraph import RoomNode
|
||
from simulator import RoomView, collect_room_views_by_room
|
||
|
||
rgb = np.ones((4, 4, 3), dtype=np.uint8) * 127
|
||
depth = np.full((4, 4), 0.5, dtype=np.float32)
|
||
|
||
room_node = RoomNode(room_id="room_a", center=np.array([4, 5, 6], dtype=np.float32), bbox_extent=[2, 2, 2])
|
||
room_nodes = [room_node]
|
||
|
||
sim = FakeSimForViews(observations_by_step={
|
||
0: {"color_sensor": rgb, "depth_sensor": depth},
|
||
})
|
||
|
||
agent_pos = np.array([4, 5, 6], dtype=np.float32)
|
||
camera_pos = np.array([4.0, 6.5, 6.0], dtype=np.float32) # sensor offset [0, 1.5, 0]
|
||
camera_rot = np.eye(3, dtype=np.float32)
|
||
|
||
agent = FakeAgentForViews()
|
||
agent.set_state(FakeAgentState(
|
||
position=agent_pos.copy(),
|
||
rotation=np.eye(3, dtype=np.float32),
|
||
sensor_states={
|
||
"depth_sensor": SimpleNamespace(
|
||
position=camera_pos.copy(),
|
||
rotation=camera_rot.copy(),
|
||
),
|
||
},
|
||
))
|
||
|
||
result = collect_room_views_by_room(
|
||
agent=agent,
|
||
sim=sim,
|
||
room_nodes=room_nodes,
|
||
views_per_room=1,
|
||
habitat_sim_module=FakeHabitatModuleForViews,
|
||
sensor_uuid="color_sensor",
|
||
depth_sensor_uuid="depth_sensor",
|
||
turn_action="turn_left",
|
||
progress_track=lambda x, desc: x,
|
||
)
|
||
|
||
room_views = result["room_a"]
|
||
assert len(room_views) == 1
|
||
|
||
item = room_views[0]
|
||
assert isinstance(item, RoomView)
|
||
assert item.rgb is rgb
|
||
assert item.depth is depth
|
||
np.testing.assert_array_equal(item.agent_position, agent_pos)
|
||
np.testing.assert_array_equal(item.agent_rotation, np.eye(3, dtype=np.float32))
|
||
# Camera fields populated from sensor_states
|
||
np.testing.assert_array_equal(item.camera_position, camera_pos)
|
||
np.testing.assert_array_equal(item.camera_rotation, camera_rot)
|
||
assert sim.steps == ["turn_left"]
|
||
|
||
|
||
def test_collect_room_views_by_room_snapshots_pose_independently():
|
||
"""Collected RoomViews must have independent snapshots of agent_rotation (not aliased)."""
|
||
from scenegraph import RoomNode
|
||
from simulator import RoomView, collect_room_views_by_room
|
||
|
||
rgb = np.ones((4, 4, 3), dtype=np.uint8) * 127
|
||
depth = np.full((4, 4), 0.5, dtype=np.float32)
|
||
|
||
room_node = RoomNode(
|
||
room_id="room_a",
|
||
center=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
bbox_extent=[2, 2, 2],
|
||
)
|
||
|
||
class _MutatingFakeSim(FakeSimForViews):
|
||
"""Fake sim that mutates agent rotation in-place on step (like real habitat)."""
|
||
|
||
def __init__(self, agent, observations_by_step=None):
|
||
super().__init__(observations_by_step)
|
||
self.agent = agent
|
||
|
||
def step(self, action):
|
||
super().step(action)
|
||
# In-place mutation – simulates habitat_sim rotating the agent's state
|
||
step_idx = len(self.steps)
|
||
self.agent._state.rotation[:] = np.full_like(
|
||
self.agent._state.rotation, float(step_idx)
|
||
)
|
||
|
||
agent = FakeAgentForViews()
|
||
sim = _MutatingFakeSim(
|
||
agent=agent,
|
||
observations_by_step={
|
||
0: {"color_sensor": rgb, "depth_sensor": depth},
|
||
1: {"color_sensor": rgb, "depth_sensor": depth},
|
||
},
|
||
)
|
||
|
||
result = collect_room_views_by_room(
|
||
agent=agent,
|
||
sim=sim,
|
||
room_nodes=[room_node],
|
||
views_per_room=2,
|
||
habitat_sim_module=FakeHabitatModuleForViews,
|
||
sensor_uuid="color_sensor",
|
||
depth_sensor_uuid="depth_sensor",
|
||
turn_action="turn_left",
|
||
progress_track=lambda x, desc: x,
|
||
)
|
||
|
||
room_views = result["room_a"]
|
||
assert len(room_views) == 2
|
||
|
||
rv0, rv1 = room_views[0], room_views[1]
|
||
|
||
# Each snapshot should reflect rotation *at capture time*
|
||
# – view 0 captured before any step → identity
|
||
identity = np.eye(3, dtype=np.float32)
|
||
np.testing.assert_array_equal(rv0.agent_rotation, identity)
|
||
|
||
# – view 1 captured after one in-place mutation → all 1.0
|
||
expected_mutated = np.full((3, 3), 1.0, dtype=np.float32)
|
||
np.testing.assert_array_equal(rv1.agent_rotation, expected_mutated)
|
||
|
||
# rv0 must NOT have been affected by the in-place mutation
|
||
np.testing.assert_array_equal(rv0.agent_rotation, identity)
|
||
|
||
# Positions are also independent snapshots
|
||
expected_pos = np.array([1.0, 2.0, 3.0], dtype=np.float32)
|
||
np.testing.assert_array_equal(rv0.agent_position, expected_pos)
|
||
np.testing.assert_array_equal(rv1.agent_position, expected_pos)
|
||
|
||
# Post-hoc mutation of one snapshot must not affect the other
|
||
rv0.agent_rotation[:] = 99.0
|
||
np.testing.assert_array_equal(rv1.agent_rotation, expected_mutated)
|
||
|
||
|
||
def test_collect_room_views_by_room_snapshots_camera_pose_independently():
|
||
"""Camera pose from sensor_states must be independently snapshotted (not aliased)."""
|
||
from types import SimpleNamespace
|
||
|
||
from scenegraph import RoomNode
|
||
from simulator import RoomView, collect_room_views_by_room
|
||
|
||
rgb = np.ones((4, 4, 3), dtype=np.uint8) * 127
|
||
depth = np.full((4, 4), 0.5, dtype=np.float32)
|
||
|
||
room_node = RoomNode(
|
||
room_id="room_a",
|
||
center=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
bbox_extent=[2, 2, 2],
|
||
)
|
||
|
||
agent_pos = np.array([1.0, 2.0, 3.0], dtype=np.float32)
|
||
camera_pos_base = np.array([1.0, 3.5, 3.0], dtype=np.float32) # sensor at +1.5 Y
|
||
camera_rot_base = np.eye(3, dtype=np.float32)
|
||
|
||
class _MutatingCameraSim(FakeSimForViews):
|
||
"""Fake sim that mutates sensor_states rotation in-place on step."""
|
||
|
||
def __init__(self, agent, observations_by_step=None):
|
||
super().__init__(observations_by_step)
|
||
self.agent = agent
|
||
|
||
def step(self, action):
|
||
super().step(action)
|
||
# Mutate the sensor rotation in-place – like real habitat might
|
||
step_idx = len(self.steps)
|
||
ds = self.agent._state.sensor_states["depth_sensor"]
|
||
ds.rotation[:] = np.full_like(ds.rotation, float(step_idx))
|
||
|
||
agent = FakeAgentForViews()
|
||
agent._state = FakeAgentState(
|
||
position=agent_pos.copy(),
|
||
rotation=np.eye(3, dtype=np.float32),
|
||
sensor_states={
|
||
"depth_sensor": SimpleNamespace(
|
||
position=camera_pos_base.copy(),
|
||
rotation=camera_rot_base.copy(),
|
||
),
|
||
},
|
||
)
|
||
|
||
sim = _MutatingCameraSim(
|
||
agent=agent,
|
||
observations_by_step={
|
||
0: {"color_sensor": rgb, "depth_sensor": depth},
|
||
1: {"color_sensor": rgb, "depth_sensor": depth},
|
||
},
|
||
)
|
||
|
||
result = collect_room_views_by_room(
|
||
agent=agent,
|
||
sim=sim,
|
||
room_nodes=[room_node],
|
||
views_per_room=2,
|
||
habitat_sim_module=FakeHabitatModuleForViews,
|
||
sensor_uuid="color_sensor",
|
||
depth_sensor_uuid="depth_sensor",
|
||
turn_action="turn_left",
|
||
progress_track=lambda x, desc: x,
|
||
)
|
||
|
||
room_views = result["room_a"]
|
||
assert len(room_views) == 2
|
||
|
||
rv0, rv1 = room_views[0], room_views[1]
|
||
|
||
# view 0 captured before any step → identity rotation
|
||
np.testing.assert_array_equal(rv0.camera_rotation, np.eye(3, dtype=np.float32))
|
||
|
||
# view 1 captured after one in-place mutation → all 1.0
|
||
expected_mutated = np.full((3, 3), 1.0, dtype=np.float32)
|
||
np.testing.assert_array_equal(rv1.camera_rotation, expected_mutated)
|
||
|
||
# rv0 rotation must NOT have been affected by in-place mutation
|
||
np.testing.assert_array_equal(rv0.camera_rotation, np.eye(3, dtype=np.float32))
|
||
|
||
# Both camera positions are the same (position not mutated in this test)
|
||
np.testing.assert_array_equal(rv0.camera_position, camera_pos_base)
|
||
np.testing.assert_array_equal(rv1.camera_position, camera_pos_base)
|
||
|
||
# Post-hoc mutation of one snapshot must not affect the other
|
||
rv0.camera_rotation[:] = 99.0
|
||
np.testing.assert_array_equal(rv1.camera_rotation, expected_mutated)
|
||
|
||
|
||
def test_flatten_room_views_preserves_existing_room_view_payloads():
|
||
"""flatten_room_views should pass through existing RoomView instances unchanged."""
|
||
from simulator import RoomView, flatten_room_views
|
||
|
||
depth = np.full((4, 4), 0.5, dtype=np.float32)
|
||
agent_pos = np.array([1.0, 2.0, 3.0], dtype=np.float32)
|
||
|
||
existing = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=np.zeros((4, 4, 3), dtype=np.uint8),
|
||
depth=depth,
|
||
agent_position=agent_pos,
|
||
agent_rotation=np.eye(3, dtype=np.float32),
|
||
)
|
||
|
||
result = flatten_room_views({"room_a": [existing]})
|
||
|
||
assert len(result) == 1
|
||
assert result[0] is existing
|
||
assert result[0].depth is depth
|
||
assert result[0].agent_position is agent_pos
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Scene graph depth positioning M1 — config + builder tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_config_accepts_bbox_depth_center_strategy():
|
||
"""Config should accept position_strategy='bbox_depth_center' with valid camera_hfov."""
|
||
from scenegraph import SceneGraphBuildConfig
|
||
|
||
config = SceneGraphBuildConfig(
|
||
position_strategy='bbox_depth_center',
|
||
camera_hfov_degrees=90.0,
|
||
)
|
||
assert config.position_strategy == 'bbox_depth_center'
|
||
assert config.camera_hfov_degrees == 90.0
|
||
|
||
|
||
def test_config_rejects_invalid_camera_hfov():
|
||
"""Config should reject camera_hfov_degrees outside (0, 180)."""
|
||
from scenegraph import SceneGraphBuildConfig
|
||
|
||
with pytest.raises(ValueError, match="camera_hfov_degrees"):
|
||
SceneGraphBuildConfig(camera_hfov_degrees=180.0)
|
||
|
||
with pytest.raises(ValueError, match="camera_hfov_degrees"):
|
||
SceneGraphBuildConfig(camera_hfov_degrees=0.0)
|
||
|
||
with pytest.raises(ValueError, match="camera_hfov_degrees"):
|
||
SceneGraphBuildConfig(camera_hfov_degrees=-10.0)
|
||
|
||
|
||
def test_scene_graph_builder_uses_bbox_depth_center_position_with_identity_pose():
|
||
"""Bbox depth center with identity rotation should project depth correctly.
|
||
|
||
Setup:
|
||
- 5x5 depth image, depth=2.0 everywhere
|
||
- bbox [2,2,2,2] → centre pixel (u=2, v=2)
|
||
- camera_position [1, 2, 3], identity rotation
|
||
- hfov=90° → fx = 5 / (2*tan(45°)) = 2.5, cx=2, cy=2
|
||
|
||
Expected camera point: [(2-2)/2.5*2, -(2-2)/2.5*2, -2] = [0, 0, -2]
|
||
Expected world position: [1, 2, 3] + [0, 0, -2] = [1, 2, 1]
|
||
"""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
depth = np.full((5, 5), 2.0, dtype=np.float32)
|
||
rgb = np.zeros((5, 5, 3), dtype=np.uint8)
|
||
|
||
cam_pos = np.array([1.0, 2.0, 3.0], dtype=np.float32)
|
||
cam_rot = np.eye(3, dtype=np.float32)
|
||
|
||
room_view = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=rgb,
|
||
depth=depth,
|
||
agent_position=cam_pos.copy(),
|
||
agent_rotation=cam_rot.copy(),
|
||
camera_position=cam_pos.copy(),
|
||
camera_rotation=cam_rot.copy(),
|
||
)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
|
||
crop = "crop_data_0"
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[2.0, 2.0, 2.0, 2.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((5, 5), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(
|
||
position_strategy='bbox_depth_center',
|
||
camera_hfov_degrees=90.0,
|
||
)
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
np.testing.assert_array_almost_equal(
|
||
obj.position,
|
||
np.array([1.0, 2.0, 1.0], dtype=np.float32),
|
||
)
|
||
assert obj.position_confidence == 1.0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Camera pose with non-zero sensor height
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_bbox_depth_center_uses_camera_pose_with_sensor_height():
|
||
"""Bbox depth center with camera pose at non-zero sensor height.
|
||
|
||
Demonstrates that the camera's sensor_height offset (Y direction) is
|
||
correctly honored: the depth back-projection uses camera_position
|
||
(which includes [0, sensor_height, 0]) instead of agent_position
|
||
(which does not).
|
||
|
||
Setup:
|
||
- camera_position [1, 3.5, 3] (e.g. agent at [1, 2, 3] + sensor_height 1.5)
|
||
- identity rotation
|
||
- depth=2.0, hfov=90°, image 5x5, bbox [2,2,2,2]
|
||
|
||
Expected camera point: [0, 0, -2]
|
||
Expected world position: [1, 3.5, 3] + [0, 0, -2] = [1, 3.5, 1]
|
||
(NOT [1, 2, 0] which would result from using agent_position [1,2,3])
|
||
"""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
depth = np.full((5, 5), 2.0, dtype=np.float32)
|
||
rgb = np.zeros((5, 5, 3), dtype=np.uint8)
|
||
|
||
room_view = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=rgb,
|
||
depth=depth,
|
||
agent_position=np.array([1.0, 2.0, 3.0], dtype=np.float32), # agent body
|
||
agent_rotation=np.eye(3, dtype=np.float32),
|
||
camera_position=np.array([1.0, 3.5, 3.0], dtype=np.float32), # sensor at +1.5 Y
|
||
camera_rotation=np.eye(3, dtype=np.float32),
|
||
)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
|
||
crop = "crop_data_0"
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[2.0, 2.0, 2.0, 2.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((5, 5), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(
|
||
position_strategy='bbox_depth_center',
|
||
camera_hfov_degrees=90.0,
|
||
)
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
# [1, 3.5, 3] + [0, 0, -2] = [1, 3.5, 1]
|
||
np.testing.assert_array_almost_equal(
|
||
obj.position,
|
||
np.array([1.0, 3.5, 1.0], dtype=np.float32),
|
||
)
|
||
assert obj.position_confidence == 1.0
|
||
|
||
|
||
def test_bbox_depth_center_falls_back_without_camera_pose():
|
||
"""Bbox depth center with only agent_pose (no camera_pose) falls back to room center.
|
||
|
||
When a RoomView has agent_position/agent_rotation but no
|
||
camera_position/camera_rotation, the builder must NOT use agent_pose
|
||
for depth positioning (because sensor offset is unknowable). The
|
||
position must fall back to room center with 0.0 confidence.
|
||
"""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
depth = np.full((5, 5), 2.0, dtype=np.float32)
|
||
rgb = np.zeros((5, 5, 3), dtype=np.uint8)
|
||
|
||
# Has agent_pose but NO camera_pose — builder must NOT use agent_pose for depth
|
||
room_view = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=rgb,
|
||
depth=depth,
|
||
agent_position=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
agent_rotation=np.eye(3, dtype=np.float32),
|
||
)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
|
||
crop = "crop_data_0"
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[2.0, 2.0, 2.0, 2.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((5, 5), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(position_strategy='bbox_depth_center')
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
# Falls back to room center with 0.0 confidence
|
||
np.testing.assert_array_equal(obj.position, np.array([10, 0, 20], dtype=np.float32))
|
||
assert obj.position_confidence == 0.0
|
||
|
||
|
||
def test_scene_graph_builder_falls_back_to_room_center_without_depth_pose():
|
||
"""Bbox depth strategy without depth/pose should fall back to room center + 0.0 confidence."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
rgb = np.zeros((32, 32, 3), dtype=np.uint8)
|
||
room_view = RoomView(room_id="room_a", view_idx=0, rgb=rgb) # no depth/pose
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
|
||
crop = "crop_data_0"
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[10.0, 20.0, 30.0, 40.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((32, 32), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(position_strategy='bbox_depth_center')
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
np.testing.assert_array_equal(obj.position, np.array([10, 0, 20], dtype=np.float32))
|
||
assert obj.position_confidence == 0.0
|
||
|
||
|
||
def test_scene_graph_builder_falls_back_on_nonfinite_depth():
|
||
"""Non-finite depth value should trigger fallback to room center + 0.0 confidence."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
depth = np.full((5, 5), np.nan, dtype=np.float32) # non-finite depth
|
||
rgb = np.zeros((5, 5, 3), dtype=np.uint8)
|
||
|
||
room_view = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=rgb,
|
||
depth=depth,
|
||
agent_position=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
agent_rotation=np.eye(3, dtype=np.float32),
|
||
)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
|
||
crop = "crop_data_0"
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[2.0, 2.0, 2.0, 2.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((5, 5), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(position_strategy='bbox_depth_center')
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
np.testing.assert_array_equal(obj.position, np.array([10, 0, 20], dtype=np.float32))
|
||
assert obj.position_confidence == 0.0
|
||
|
||
|
||
def test_bbox_depth_center_falls_back_on_invalid_bbox_length():
|
||
"""Bbox with wrong length should fall back to room center + 0.0 confidence."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
depth = np.full((5, 5), 2.0, dtype=np.float32)
|
||
rgb = np.zeros((5, 5, 3), dtype=np.uint8)
|
||
|
||
room_view = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=rgb,
|
||
depth=depth,
|
||
agent_position=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
agent_rotation=np.eye(3, dtype=np.float32),
|
||
)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
crop = "crop_data_0"
|
||
|
||
# bbox with length 2 instead of 4
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[10.0, 20.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((5, 5), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(position_strategy='bbox_depth_center')
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
np.testing.assert_array_equal(obj.position, np.array([10, 0, 20], dtype=np.float32))
|
||
assert obj.position_confidence == 0.0
|
||
|
||
|
||
def test_bbox_depth_center_falls_back_on_nonfinite_bbox():
|
||
"""Bbox with non-finite values should fall back to room center + 0.0 confidence."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
depth = np.full((5, 5), 2.0, dtype=np.float32)
|
||
rgb = np.zeros((5, 5, 3), dtype=np.uint8)
|
||
|
||
room_view = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=rgb,
|
||
depth=depth,
|
||
agent_position=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
agent_rotation=np.eye(3, dtype=np.float32),
|
||
)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
crop = "crop_data_0"
|
||
|
||
# bbox with NaN value
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[float('nan'), 2.0, 2.0, 2.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((5, 5), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(position_strategy='bbox_depth_center')
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
np.testing.assert_array_equal(obj.position, np.array([10, 0, 20], dtype=np.float32))
|
||
assert obj.position_confidence == 0.0
|
||
|
||
|
||
def test_bbox_depth_center_falls_back_on_invalid_depth():
|
||
"""Depth without ndim attribute (e.g. plain list) should fall back."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
# Plain list — no .ndim attribute; old code would AttributeError
|
||
depth = [1.0, 2.0]
|
||
rgb = np.zeros((5, 5, 3), dtype=np.uint8)
|
||
|
||
room_view = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=rgb,
|
||
depth=depth,
|
||
agent_position=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
agent_rotation=np.eye(3, dtype=np.float32),
|
||
)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
crop = "crop_data_0"
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[2.0, 2.0, 2.0, 2.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((5, 5), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(position_strategy='bbox_depth_center')
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
np.testing.assert_array_equal(obj.position, np.array([10, 0, 20], dtype=np.float32))
|
||
assert obj.position_confidence == 0.0
|
||
|
||
|
||
def test_bbox_depth_center_falls_back_on_invalid_agent_position():
|
||
"""Agent position with wrong shape should fall back to room center."""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
depth = np.full((5, 5), 2.0, dtype=np.float32)
|
||
rgb = np.zeros((5, 5, 3), dtype=np.uint8)
|
||
|
||
# Position with shape (2,) — can't reshape to (3,)
|
||
room_view = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=rgb,
|
||
depth=depth,
|
||
agent_position=np.array([1.0, 2.0]),
|
||
agent_rotation=np.eye(3, dtype=np.float32),
|
||
)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
crop = "crop_data_0"
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[2.0, 2.0, 2.0, 2.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((5, 5), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(position_strategy='bbox_depth_center')
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
np.testing.assert_array_equal(obj.position, np.array([10, 0, 20], dtype=np.float32))
|
||
assert obj.position_confidence == 0.0
|
||
|
||
|
||
def test_bbox_depth_center_uses_quaternion_like_rotation():
|
||
"""Rotation with transform_vector (quaternion-like) using non-identity mapping.
|
||
|
||
FakeQuaternionRotation implements a 90° Y-rotation:
|
||
basis x → z, basis y → y, basis z → -x
|
||
→ matrix = [[0, 0, -1], [0, 1, 0], [1, 0, 0]]
|
||
|
||
Camera point (u=2, v=2, depth=2.0): [0, 0, -2]
|
||
Rotated: [2, 0, 0]
|
||
World: [1, 2, 3] + [2, 0, 0] = [3, 2, 3]
|
||
"""
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
class FakeQuaternionRotation:
|
||
def transform_vector(self, v):
|
||
v = np.asarray(v, dtype=np.float32).ravel()
|
||
# 90° Y-rotation: x→[0,0,1], y→[0,1,0], z→[-1,0,0]
|
||
if np.allclose(v, [1, 0, 0]):
|
||
return np.array([0, 0, 1], dtype=np.float32)
|
||
if np.allclose(v, [0, 1, 0]):
|
||
return np.array([0, 1, 0], dtype=np.float32)
|
||
if np.allclose(v, [0, 0, 1]):
|
||
return np.array([-1, 0, 0], dtype=np.float32)
|
||
return v.copy()
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
depth = np.full((5, 5), 2.0, dtype=np.float32)
|
||
rgb = np.zeros((5, 5, 3), dtype=np.uint8)
|
||
|
||
room_view = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=rgb,
|
||
depth=depth,
|
||
agent_position=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
agent_rotation=FakeQuaternionRotation(),
|
||
camera_position=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
camera_rotation=FakeQuaternionRotation(),
|
||
)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
|
||
crop = "crop_data_0"
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[2.0, 2.0, 2.0, 2.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((5, 5), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(
|
||
position_strategy='bbox_depth_center',
|
||
camera_hfov_degrees=90.0,
|
||
)
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
np.testing.assert_array_almost_equal(
|
||
obj.position,
|
||
np.array([3.0, 2.0, 3.0], dtype=np.float32),
|
||
)
|
||
assert obj.position_confidence == 1.0
|
||
|
||
|
||
|
||
|
||
def test_bbox_depth_center_uses_numpy_quaternion_rotation(monkeypatch):
|
||
"""quaternion.as_rotation_matrix path works via monkeypatched quaternion module.
|
||
|
||
Monkeypatches sys.modules['quaternion'] with a fake module where
|
||
as_rotation_matrix returns a non-identity 3x3 for a sentinel quaternion object.
|
||
|
||
Rotation: 90° Y-rotation → matrix = [[0,0,-1],[0,1,0],[1,0,0]]
|
||
Camera point (u=2, v=2, depth=2.0): [0, 0, -2]
|
||
Rotated: [2, 0, 0]
|
||
World: [1, 2, 3] + [2, 0, 0] = [3, 2, 3]
|
||
"""
|
||
import types
|
||
|
||
from scenegraph import RoomNode, SceneGraphBuilder, SceneGraphBuildConfig
|
||
from simulator import RoomView
|
||
|
||
class FakeQuaternion:
|
||
"""Sentinel quaternion object — no .rotation_matrix or transform_vector."""
|
||
pass
|
||
|
||
sentinel_q = FakeQuaternion()
|
||
|
||
fake_module = types.ModuleType("quaternion")
|
||
|
||
def as_rotation_matrix(q):
|
||
assert q is sentinel_q
|
||
return np.array([[0, 0, -1],
|
||
[0, 1, 0],
|
||
[1, 0, 0]], dtype=np.float32)
|
||
|
||
fake_module.as_rotation_matrix = as_rotation_matrix
|
||
monkeypatch.setitem(sys.modules, "quaternion", fake_module)
|
||
|
||
room_node = RoomNode(room_id="room_a", center=[10, 0, 20], bbox_extent=[5, 3, 5])
|
||
room_nodes = [room_node]
|
||
|
||
depth = np.full((5, 5), 2.0, dtype=np.float32)
|
||
rgb = np.zeros((5, 5, 3), dtype=np.uint8)
|
||
|
||
room_view = RoomView(
|
||
room_id="room_a",
|
||
view_idx=0,
|
||
rgb=rgb,
|
||
depth=depth,
|
||
agent_position=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
agent_rotation=sentinel_q,
|
||
camera_position=np.array([1.0, 2.0, 3.0], dtype=np.float32),
|
||
camera_rotation=sentinel_q,
|
||
)
|
||
room_views = [room_view]
|
||
|
||
text_labels = ["a chair"]
|
||
|
||
hash_bits = torch.zeros(1, 512, dtype=torch.uint8)
|
||
hash_bits[0, 3] = 1
|
||
|
||
crop = "crop_data_0"
|
||
|
||
debug_meta_entry = {
|
||
"selected_indices": [0],
|
||
"boxes_xyxy": [[2.0, 2.0, 2.0, 2.0]],
|
||
"scores": [0.91],
|
||
"labels": ["a chair"],
|
||
"masks": [np.zeros((5, 5), dtype=np.uint8)],
|
||
"num_selected": 1,
|
||
}
|
||
|
||
output = SimpleNamespace(
|
||
cropped_images=[crop],
|
||
hash_bits=hash_bits,
|
||
debug_meta=[debug_meta_entry],
|
||
)
|
||
|
||
pipeline = FakePipeline(output=output, calls=[])
|
||
config = SceneGraphBuildConfig(
|
||
position_strategy='bbox_depth_center',
|
||
camera_hfov_degrees=90.0,
|
||
)
|
||
builder = SceneGraphBuilder(pipeline=pipeline, config=config)
|
||
|
||
graph, artifacts = builder.build_from_room_views(
|
||
room_nodes=room_nodes,
|
||
room_views=room_views,
|
||
text_labels=text_labels,
|
||
)
|
||
|
||
obj = graph.objects["room_a_v000_m00"]
|
||
np.testing.assert_array_almost_equal(
|
||
obj.position,
|
||
np.array([3.0, 2.0, 3.0], dtype=np.float32),
|
||
)
|
||
assert obj.position_confidence == 1.0
|