Interactive visualization

This tutorial introduces user interaction features of the visualizer window.

 27# examples/python/visualization/interactive_visualization.py
 28
 29import numpy as np
 30import copy
 31import open3d as o3d
 32
 33
 34def demo_crop_geometry():
 35    print("Demo for manual geometry cropping")
 36    print(
 37        "1) Press 'Y' twice to align geometry with negative direction of y-axis"
 38    )
 39    print("2) Press 'K' to lock screen and to switch to selection mode")
 40    print("3) Drag for rectangle selection,")
 41    print("   or use ctrl + left click for polygon selection")
 42    print("4) Press 'C' to get a selected geometry and to save it")
 43    print("5) Press 'F' to switch to freeview mode")
 44    pcd_data = o3d.data.DemoICPPointClouds()
 45    pcd = o3d.io.read_point_cloud(pcd_data.paths[0])
 46    o3d.visualization.draw_geometries_with_editing([pcd])
 47
 48
 49def draw_registration_result(source, target, transformation):
 50    source_temp = copy.deepcopy(source)
 51    target_temp = copy.deepcopy(target)
 52    source_temp.paint_uniform_color([1, 0.706, 0])
 53    target_temp.paint_uniform_color([0, 0.651, 0.929])
 54    source_temp.transform(transformation)
 55    o3d.visualization.draw_geometries([source_temp, target_temp])
 56
 57
 58def pick_points(pcd):
 59    print("")
 60    print(
 61        "1) Please pick at least three correspondences using [shift + left click]"
 62    )
 63    print("   Press [shift + right click] to undo point picking")
 64    print("2) After picking points, press 'Q' to close the window")
 65    vis = o3d.visualization.VisualizerWithEditing()
 66    vis.create_window()
 67    vis.add_geometry(pcd)
 68    vis.run()  # user picks points
 69    vis.destroy_window()
 70    print("")
 71    return vis.get_picked_points()
 72
 73
 74def demo_manual_registration():
 75    print("Demo for manual ICP")
 76    pcd_data = o3d.data.DemoICPPointClouds()
 77    source = o3d.io.read_point_cloud(pcd_data.paths[0])
 78    target = o3d.io.read_point_cloud(pcd_data.paths[2])
 79    print("Visualization of two point clouds before manual alignment")
 80    draw_registration_result(source, target, np.identity(4))
 81
 82    # pick points from two point clouds and builds correspondences
 83    picked_id_source = pick_points(source)
 84    picked_id_target = pick_points(target)
 85    assert (len(picked_id_source) >= 3 and len(picked_id_target) >= 3)
 86    assert (len(picked_id_source) == len(picked_id_target))
 87    corr = np.zeros((len(picked_id_source), 2))
 88    corr[:, 0] = picked_id_source
 89    corr[:, 1] = picked_id_target
 90
 91    # estimate rough transformation using correspondences
 92    print("Compute a rough transform using the correspondences given by user")
 93    p2p = o3d.pipelines.registration.TransformationEstimationPointToPoint()
 94    trans_init = p2p.compute_transformation(source, target,
 95                                            o3d.utility.Vector2iVector(corr))
 96
 97    # point-to-point ICP for refinement
 98    print("Perform point-to-point ICP refinement")
 99    threshold = 0.03  # 3cm distance threshold
100    reg_p2p = o3d.pipelines.registration.registration_icp(
101        source, target, threshold, trans_init,
102        o3d.pipelines.registration.TransformationEstimationPointToPoint())
103    draw_registration_result(source, target, reg_p2p.transformation)
104    print("")
105
106
107if __name__ == "__main__":
108    demo_crop_geometry()
109    demo_manual_registration()

This script executes two applications of user interaction: demo_crop_geometry and demo_manual_registration.

Crop geometry

37        "1) Press 'Y' twice to align geometry with negative direction of y-axis"
38    )
39    print("2) Press 'K' to lock screen and to switch to selection mode")
40    print("3) Drag for rectangle selection,")
41    print("   or use ctrl + left click for polygon selection")
42    print("4) Press 'C' to get a selected geometry and to save it")
43    print("5) Press 'F' to switch to freeview mode")
44    pcd_data = o3d.data.DemoICPPointClouds()
45    pcd = o3d.io.read_point_cloud(pcd_data.paths[0])
46    o3d.visualization.draw_geometries_with_editing([pcd])
47
48
49def draw_registration_result(source, target, transformation):
50    source_temp = copy.deepcopy(source)
51    target_temp = copy.deepcopy(target)

This function simply reads a point cloud and calls draw_geometries_with_editing. This function provides vertex selection and cropping.

Note

Open3D has a VisualizerWithEditing class that inherits Visualizer class. It adds graphic user interaction features. Likewise examples in Customized visualization, VisualizerWithEditing() can be explicitly used instead of draw_geometries_with_editing([pcd]).

Once a geometry is displayed, press Y twice to align geometry with negative direction of y-axis. After adjusting viewing orientation, press K to lock screen and to switch to the selection mode.

../../_images/crop_lock.png

Tip

The practical step for selecting area is to align the geometry with arbitrary axis using orthographic projection model. This trick makes selection easier, because it avoids self-occlusion hassle due to perspective projection.

To select a region, use either mouse drag (rectangle selection) or ctrl + left mouse click (polygon selection). The below example shows a selected area using a polygon.

../../_images/crop_selection.png

Note that the selected area is dark shaded. To keep the selected area and discard the rest, press C. A dialog box appears, which can be used to save the cropped geometry. The cropping result is shown after saving.

../../_images/crop_save.png ../../_images/crop_result.png

To finish selection mode, press F to switch to freeview mode.

../../_images/crop_freeview.png

Manual registration

Select correspondences

The following script registers two point clouds using point-to-point ICP. It gets initial alignment via user interaction.

61        "1) Please pick at least three correspondences using [shift + left click]"
62    )
63    print("   Press [shift + right click] to undo point picking")
64    print("2) After picking points, press 'Q' to close the window")
65    vis = o3d.visualization.VisualizerWithEditing()
66    vis.create_window()
67    vis.add_geometry(pcd)
68    vis.run()  # user picks points
69    vis.destroy_window()
70    print("")
71    return vis.get_picked_points()
72
73
74def demo_manual_registration():
75    print("Demo for manual ICP")
76    pcd_data = o3d.data.DemoICPPointClouds()

The script reads two point clouds, and visualizes the point clouds before alignment.

../../_images/manual_icp_initial.png
52    source_temp.paint_uniform_color([1, 0.706, 0])
53    target_temp.paint_uniform_color([0, 0.651, 0.929])
54    source_temp.transform(transformation)
55    o3d.visualization.draw_geometries([source_temp, target_temp])
56
57
58def pick_points(pcd):
59    print("")
60    print(

The function pick_points(pcd) makes an instance of VisualizerWithEditing. To mimic draw_geometries, it creates windows, adds the geometry, visualizes the geometry, and then terminates. A novel interface function from VisualizerWithEditing is get_picked_points() that returns the indices of user-picked vertices.

To pick a vertex, press shift + left click on a window. If a vertex is selected, the visualizer window overlays a sphere on a selected vertex. For example, after picking three vertices in the source point cloud, it shows:

../../_images/manual_icp_source.png

This will print:

Picked point #58481 (2.14, 1.56, 1.53) to add in queue.
Picked point #77321 (2.86, 1.92, 1.09) to add in queue.
Picked point #42639 (3.28, 1.53, 1.45) to add in queue.

Press Q to close the window. The next step is to pick the same correspondences in the target point cloud. The color of the sphere helps to identify the same correspondence.

../../_images/manual_icp_target.png

This will print:

Picked point #54028 (1.62, 1.81, 1.23) to add in queue.
Picked point #97115 (2.45, 2.19, 1.11) to add in queue.
Picked point #47467 (2.75, 1.71, 1.45) to add in queue.

Tip

To get a good registration result, try to pick more than three points that are well-distributed in the scene. Using a vertex in the corner region is a good way to easily pick the right correspondence.

Registration using user correspondences

 77    source = o3d.io.read_point_cloud(pcd_data.paths[0])
 78    target = o3d.io.read_point_cloud(pcd_data.paths[2])
 79    print("Visualization of two point clouds before manual alignment")
 80    draw_registration_result(source, target, np.identity(4))
 81
 82    # pick points from two point clouds and builds correspondences
 83    picked_id_source = pick_points(source)
 84    picked_id_target = pick_points(target)
 85    assert (len(picked_id_source) >= 3 and len(picked_id_target) >= 3)
 86    assert (len(picked_id_source) == len(picked_id_target))
 87    corr = np.zeros((len(picked_id_source), 2))
 88    corr[:, 0] = picked_id_source
 89    corr[:, 1] = picked_id_target
 90
 91    # estimate rough transformation using correspondences
 92    print("Compute a rough transform using the correspondences given by user")
 93    p2p = o3d.pipelines.registration.TransformationEstimationPointToPoint()
 94    trans_init = p2p.compute_transformation(source, target,
 95                                            o3d.utility.Vector2iVector(corr))
 96
 97    # point-to-point ICP for refinement
 98    print("Perform point-to-point ICP refinement")
 99    threshold = 0.03  # 3cm distance threshold
100    reg_p2p = o3d.pipelines.registration.registration_icp(
101        source, target, threshold, trans_init,
102        o3d.pipelines.registration.TransformationEstimationPointToPoint())
103    draw_registration_result(source, target, reg_p2p.transformation)
104    print("")
105
106
107if __name__ == "__main__":
108    demo_crop_geometry()
109    demo_manual_registration()

The later part of the demo computes an initial transformation based on the user-provided correspondences. This script builds pairs of correspondences using Vector2iVector(corr). It utilizes TransformationEstimationPointToPoint.compute_transformation to compute the initial transformation from the correspondences. The initial transformation is refined using registration_icp.

The registration result is as follows:

../../_images/manual_icp_alignment.png