Mini-Nav/notebooks/verification.py

# /// script
# requires-python = ">=3.13"
# dependencies = [
#     "marimo>=0.21.1",
#     "pyzmq>=27.1.0",
# ]
# ///

import marimo

__generated_with = "0.21.1"
app = marimo.App(width="medium", app_title="Pipeline Verification")


@app.cell
def import_packages():
    from io import BytesIO

    import marimo as mo
    import numpy as np
    import polars as pl
    from habitat.utils.visualizations import maps
    from matplotlib import pyplot as plt
    from PIL import Image

    from compressors.pipeline import HashPipeline
    from scenegraph import ObjectNode, RoomNode, SimpleSceneGraph
    from simulator import (
        HabitatSimulatorConfig,
        TopDownSceneElements,
        collect_room_views_by_room,
        create_habitat_simulator,
        render_topdown_scene_map,
    )
    from utils.image import extract_masked_region, segment_image

    return (
        HabitatSimulatorConfig,
        HashPipeline,
        Image,
        ObjectNode,
        RoomNode,
        SimpleSceneGraph,
        TopDownSceneElements,
        collect_room_views_by_room,
        create_habitat_simulator,
        extract_masked_region,
        maps,
        mo,
        np,
        pl,
        plt,
        render_topdown_scene_map,
        segment_image,
    )


@app.cell
def setup_verification_context(
    HabitatSimulatorConfig,
    RoomNode,
    create_habitat_simulator,
    np,
):
    scene_path = "data/scene_datasets/habitat-test-scenes/skokloster-castle.glb"
    image_size = 256
    num_rooms = 4
    views_per_room = 6
    meters_per_pixel = 0.05

    sam_max_masks = 5
    sam_min_area = 32 * 32
    hash_bits = 512
    pipeline_batch_size = 64

    sim, agent = create_habitat_simulator(
        HabitatSimulatorConfig(
            scene_path=scene_path,
            views_per_room=views_per_room,
            image_size=image_size,
            sensor_height=1.5,
            move_forward_step=0.25,
            enable_physics=False,
        )
    )

    room_nodes = []
    for idx in range(num_rooms):
        point = sim.pathfinder.get_random_navigable_point()
        room_nodes.append(
            RoomNode(
                room_id=f"room_{idx:02d}",
                center=np.asarray(point, dtype=np.float32),
                bbox_extent=np.asarray([1.5, 2.0, 1.5], dtype=np.float32),
            )
        )

    print("Sampled room centers:")
    for node in room_nodes:
        print(node.room_id, node.center)
    return (
        agent,
        hash_bits,
        meters_per_pixel,
        pipeline_batch_size,
        room_nodes,
        sam_max_masks,
        sam_min_area,
        sim,
        views_per_room,
    )


@app.cell
def render_topdown_room_map(
    TopDownSceneElements,
    maps,
    meters_per_pixel,
    plt,
    render_topdown_scene_map,
    room_nodes,
    sim,
):
    render_topdown_scene_map(
        pathfinder=sim.pathfinder,
        elements=TopDownSceneElements(room_nodes=room_nodes),
        meters_per_pixel=meters_per_pixel,
        maps_module=maps,
        plt_module=plt,
    )
    return


@app.cell
def build_scene_graph_pipeline(
    HashPipeline,
    Image,
    ObjectNode,
    SimpleSceneGraph,
    agent,
    collect_room_views_by_room,
    extract_masked_region,
    hash_bits,
    mo,
    np,
    pipeline_batch_size,
    room_nodes,
    sam_max_masks,
    sam_min_area,
    segment_image,
    sim,
    views_per_room,
):
    all_room_views = collect_room_views_by_room(
        agent=agent,
        sim=sim,
        room_nodes=room_nodes,
        views_per_room=views_per_room,
    )

    hash_pipeline = HashPipeline(
        dino_model="facebook/dinov2-large",
        sam_model="facebook/sam2.1-hiera-large",
        sam_min_mask_area=sam_min_area,
        sam_max_masks=sam_max_masks,
        hash_bits=hash_bits,
    )

    scene_graph = SimpleSceneGraph(
        rooms={room.room_id: room for room in room_nodes},
        objects={},
    )
    total_masks = 0
    object_index = 0

    room_view_dataset = [
        (room_id, view_idx, rgb)
        for room_id, views in all_room_views.items()
        for view_idx, rgb in enumerate(views)
    ]
    object_dataset = []

    for room_id, _view_idx, rgb in mo.status.progress_bar(
        room_view_dataset,
        title="Building object dataset",
        subtitle="Running SAM segmentation",
        show_eta=True,
        show_rate=True,
    ):
        rgb3 = rgb[..., :3] if rgb.shape[-1] > 3 else rgb
        image = Image.fromarray(rgb3.astype(np.uint8))

        masks = segment_image(
            hash_pipeline.mask_generator,
            image,
            min_area=hash_pipeline.sam_min_mask_area,
            max_masks=hash_pipeline.sam_max_masks,
            points_per_batch=hash_pipeline.sam_points_per_batch,
        )
        total_masks += len(masks)

        for mask in masks:
            masked_image = extract_masked_region(image, mask["segment"])
            object_dataset.append((room_id, mask["bbox"], masked_image))

    if object_dataset:
        masked_images = [item[2] for item in object_dataset]
        batched_bits = hash_pipeline.forward_dataset(
            masked_images,
            batch_size=pipeline_batch_size,
            apply_sam=False,
        )
        if len(batched_bits) != len(object_dataset):
            raise RuntimeError(
                "Batch output size mismatch between masked images and hash outputs."
            )
    else:
        batched_bits = []

    for ob_idx, (room_id, bbox, _) in enumerate(object_dataset):
        bits = batched_bits[ob_idx]
        obj_center = np.array(
            [bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2, 0.0],
            dtype=np.float32,
        )

        obj_id = f"obj_{object_index:04d}"
        object_index += 1

        bits_array = np.asarray(bits.detach().cpu().numpy()).reshape(-1)
        if bits_array.size == 512:
            bits_binary = (bits_array > 0).astype(np.uint8)
            hash_bytes = np.packbits(bits_binary).tobytes()
        elif bits_array.size == 64:
            hash_bytes = bits_array.astype(np.uint8).tobytes()
        else:
            raise ValueError(
                f"Unexpected hash length: {bits_array.size}. Expected 512 bits or 64 bytes."
            )

        scene_graph.objects[obj_id] = ObjectNode(
            obj_id=obj_id,
            room_id=room_id,
            position=obj_center,
            visual_hash=hash_bytes,
            semantic_hash=hash_bytes,
            hit_count=1,
            last_seen_frame=0,
        )

    print(f"Total objects created: {len(scene_graph.objects)}")
    print(f"Total processed masks: {total_masks}")
    return (scene_graph,)


@app.cell
def build_room_and_object_tables(pl, scene_graph):
    room_rows = [
        {
            "room_id": room.room_id,
            "center_x": float(room.center[0]),
            "center_y": float(room.center[1]),
            "center_z": float(room.center[2]),
            "bbox_dx": float(room.bbox_extent[0]),
            "bbox_dy": float(room.bbox_extent[1]),
            "bbox_dz": float(room.bbox_extent[2]),
        }
        for room in scene_graph.rooms.values()
    ]

    object_rows = [
        {
            "obj_id": obj.obj_id,
            "room_id": obj.room_id,
            "last_seen_frame": int(obj.last_seen_frame),
            "hit_count": int(obj.hit_count),
            "visual_hash": obj.visual_hash.hex(),
            "semantic_hash": obj.semantic_hash.hex(),
        }
        for obj in scene_graph.objects.values()
    ]

    rooms_table = pl.DataFrame(room_rows)
    objects_table = pl.DataFrame(object_rows)
    return objects_table, rooms_table


@app.cell
def upload_query_image(mo):
    file_upload = mo.ui.file(
        filetypes=["image/*"],
        kind="area",
        label="Upload a query image",
    )
    file_upload
    return (file_upload,)


@app.cell
def _(file_upload, mo):
    upload_image = None
    if file_upload.value:
        upload_image = mo.image(file_upload.contents(), alt="Uploaded query image")

    # Build a grid.
    upload_image if upload_image is not None else mo.md("No image uploaded yet.")
    return


if __name__ == "__main__":
    app.run()