def detect(self, color_im, depth_im, cfg, camera_intr=None,
               T_camera_world=None,
               vis_foreground=False, vis_segmentation=False, segmask=None):
        """
        Detects all relevant objects in an rgbd image pair using foreground masking.

        Parameters
        ----------
        color_im : :obj:`ColorImage`
            color image for detection
        depth_im : :obj:`DepthImage`
            depth image for detection (corresponds to color image)
        cfg : :obj:`YamlConfig`
            parameters of detection function
        camera_intr : :obj:`CameraIntrinsics`
            intrinsics of the camera
        T_camera_world : :obj:`autolab_core.RigidTransform`
            registration of the camera to world frame
        segmask : :obj:`BinaryImage`
            optional segmask of invalid pixels

        Returns
        ------
        :obj:`list` of :obj:`RgbdDetection`
            all detections in the image
        """
        # read params
        foreground_mask_tolerance = cfg['foreground_mask_tolerance']
        min_contour_area = cfg['min_contour_area']
        max_contour_area = cfg['max_contour_area']
        min_box_area = cfg['min_box_area']
        max_box_area = cfg['max_box_area']
        box_padding_px = cfg['box_padding_px']
        crop_height = cfg['image_height']
        crop_width = cfg['image_width']
        depth_grad_thresh = cfg['depth_grad_thresh']

        w = cfg['filter_dim']

        half_crop_height = float(crop_height) / 2
        half_crop_width = float(crop_width) / 2
        half_crop_dims = np.array([half_crop_height, half_crop_width])

        fill_depth = np.max(depth_im.data[depth_im.data > 0])

        kinect2_denoising = False
        if 'kinect2_denoising' in cfg.keys() and cfg['kinect2_denoising']:
            kinect2_denoising = True
            depth_offset = cfg['kinect2_noise_offset']
            max_depth = cfg['kinect2_noise_max_depth']

        # mask image using background detection
        bgmodel = color_im.background_model()
        binary_im = color_im.foreground_mask(foreground_mask_tolerance, bgmodel=bgmodel)

        # filter the image
        y, x = np.ogrid[-w/2+1:w/2+1, -w/2+1:w/2+1]
        mask = x*x + y*y <= w/2*w/2
        filter_struct = np.zeros([w,w]).astype(np.uint8)
        filter_struct[mask] = 1
        binary_im_filtered = binary_im.apply(snm.grey_closing, structure=filter_struct)

        # find all contours
        contours = binary_im_filtered.find_contours(min_area=min_contour_area, max_area=max_contour_area)

        if vis_foreground:
            plt.figure()
            plt.subplot(1,2,1)
            plt.imshow(color_im.data)
            plt.axis('off')
            plt.subplot(1,2,2)
            plt.imshow(binary_im_filtered.data, cmap=plt.cm.gray)
            plt.axis('off')
            plt.show()

        # threshold gradients of depth
        depth_im = depth_im.threshold_gradients(depth_grad_thresh)

        # convert contours to detections
        detections = []
        for contour in contours:
            orig_box = contour.bounding_box
            if orig_box.area > min_box_area and orig_box.area < max_box_area:
                # convert orig bounding box to query bounding box
                min_pt = orig_box.center - half_crop_dims
                max_pt = orig_box.center + half_crop_dims
                query_box = Box(min_pt, max_pt, frame=orig_box.frame)

                # segment color to get refined detection
                color_thumbnail = color_im.crop(query_box.height, query_box.width, query_box.ci, query_box.cj)
                binary_thumbnail, segment_thumbnail, query_box = self._segment_color(color_thumbnail, query_box, bgmodel, cfg, vis_segmentation=vis_segmentation)
                if binary_thumbnail is None:
                    continue
            else:
                # otherwise take original bounding box
                query_box = Box(contour.bounding_box.min_pt - box_padding_px,
                                contour.bounding_box.max_pt + box_padding_px,
                                frame = contour.bounding_box.frame)

                binary_thumbnail = binary_im_filtered.crop(query_box.height, query_box.width, query_box.ci, query_box.cj)

            # crop to get thumbnails
            color_thumbnail = color_im.crop(query_box.height, query_box.width, query_box.ci, query_box.cj)
            depth_thumbnail = depth_im.crop(query_box.height, query_box.width, query_box.ci, query_box.cj)
            thumbnail_intr = camera_intr
            if camera_intr is not None:
                thumbnail_intr = camera_intr.crop(query_box.height, query_box.width, query_box.ci, query_box.cj)

            # fix depth thumbnail
            depth_thumbnail = depth_thumbnail.replace_zeros(fill_depth)
            if kinect2_denoising:
                depth_data = depth_thumbnail.data
                min_depth = np.min(depth_data)
                binary_mask_data = binary_thumbnail.data
                depth_mask_data = depth_thumbnail.mask_binary(binary_thumbnail).data
                depth_mask_data += depth_offset
                depth_data[binary_mask_data > 0] = depth_mask_data[binary_mask_data > 0]
                depth_thumbnail = DepthImage(depth_data, depth_thumbnail.frame)

            # append to detections
            detections.append(RgbdDetection(color_thumbnail,
                                            depth_thumbnail,
                                            query_box,
                                            binary_thumbnail=binary_thumbnail,
                                            contour=contour,
                                            camera_intr=thumbnail_intr))

        return detections
def detect ( self , color_im , depth_im , cfg , camera_intr = None , T_camera_world = None , vis_foreground = False , vis_segmentation = False , segmask = None ) : foreground_mask_tolerance = cfg [ 'foreground_mask_tolerance' ] min_contour_area = cfg [ 'min_contour_area' ] max_contour_area = cfg [ 'max_contour_area' ] min_box_area = cfg [ 'min_box_area' ] max_box_area = cfg [ 'max_box_area' ] box_padding_px = cfg [ 'box_padding_px' ] crop_height = cfg [ 'image_height' ] crop_width = cfg [ 'image_width' ] depth_grad_thresh = cfg [ 'depth_grad_thresh' ] w = cfg [ 'filter_dim' ] half_crop_height = float ( crop_height ) / 2 half_crop_width = float ( crop_width ) / 2 half_crop_dims = np . array ( [ half_crop_height , half_crop_width ] ) fill_depth = np . max ( depth_im . data [ depth_im . data > 0 ] ) kinect2_denoising = False if 'kinect2_denoising' in cfg . keys ( ) and cfg [ 'kinect2_denoising' ] : kinect2_denoising = True depth_offset = cfg [ 'kinect2_noise_offset' ] max_depth = cfg [ 'kinect2_noise_max_depth' ] bgmodel = color_im . background_model ( ) binary_im = color_im . foreground_mask ( foreground_mask_tolerance , bgmodel = bgmodel ) y , x = np . ogrid [ - w / 2 + 1 : w / 2 + 1 , - w / 2 + 1 : w / 2 + 1 ] mask = x * x + y * y <= w / 2 * w / 2 filter_struct = np . zeros ( [ w , w ] ) . astype ( np . uint8 ) filter_struct [ mask ] = 1 binary_im_filtered = binary_im . apply ( snm . grey_closing , structure = filter_struct ) contours = binary_im_filtered . find_contours ( min_area = min_contour_area , max_area = max_contour_area ) if vis_foreground : plt . figure ( ) plt . subplot ( 1 , 2 , 1 ) plt . imshow ( color_im . data ) plt . axis ( 'off' ) plt . subplot ( 1 , 2 , 2 ) plt . imshow ( binary_im_filtered . data , cmap = plt . cm . gray ) plt . axis ( 'off' ) plt . show ( ) depth_im = depth_im . threshold_gradients ( depth_grad_thresh ) detections = [ ] for contour in contours : orig_box = contour . bounding_box if orig_box . area > min_box_area and orig_box . area < max_box_area : min_pt = orig_box . center - half_crop_dims max_pt = orig_box . center + half_crop_dims query_box = Box ( min_pt , max_pt , frame = orig_box . frame ) color_thumbnail = color_im . crop ( query_box . height , query_box . width , query_box . ci , query_box . cj ) binary_thumbnail , segment_thumbnail , query_box = self . _segment_color ( color_thumbnail , query_box , bgmodel , cfg , vis_segmentation = vis_segmentation ) if binary_thumbnail is None : continue else : query_box = Box ( contour . bounding_box . min_pt - box_padding_px , contour . bounding_box . max_pt + box_padding_px , frame = contour . bounding_box . frame ) binary_thumbnail = binary_im_filtered . crop ( query_box . height , query_box . width , query_box . ci , query_box . cj ) color_thumbnail = color_im . crop ( query_box . height , query_box . width , query_box . ci , query_box . cj ) depth_thumbnail = depth_im . crop ( query_box . height , query_box . width , query_box . ci , query_box . cj ) thumbnail_intr = camera_intr if camera_intr is not None : thumbnail_intr = camera_intr . crop ( query_box . height , query_box . width , query_box . ci , query_box . cj ) depth_thumbnail = depth_thumbnail . replace_zeros ( fill_depth ) if kinect2_denoising : depth_data = depth_thumbnail . data min_depth = np . min ( depth_data ) binary_mask_data = binary_thumbnail . data depth_mask_data = depth_thumbnail . mask_binary ( binary_thumbnail ) . data depth_mask_data += depth_offset depth_data [ binary_mask_data > 0 ] = depth_mask_data [ binary_mask_data > 0 ] depth_thumbnail = DepthImage ( depth_data , depth_thumbnail . frame ) detections . append ( RgbdDetection ( color_thumbnail , depth_thumbnail , query_box , binary_thumbnail = binary_thumbnail , contour = contour , camera_intr = thumbnail_intr ) ) return detections
Detects all relevant objects in an rgbd image pair using foreground masking.

        Parameters
        ----------
        color_im : :obj:`ColorImage`
            color image for detection
        depth_im : :obj:`DepthImage`
            depth image for detection (corresponds to color image)
        cfg : :obj:`YamlConfig`
            parameters of detection function
        camera_intr : :obj:`CameraIntrinsics`
            intrinsics of the camera
        T_camera_world : :obj:`autolab_core.RigidTransform`
            registration of the camera to world frame
        segmask : :obj:`BinaryImage`
            optional segmask of invalid pixels

        Returns
        ------
        :obj:`list` of :obj:`RgbdDetection`
            all detections in the image