deepsport_utilities.ds.instants_dataset.views_dataset API documentation

# Module `deepsport_utilities.ds.instants_dataset.views_dataset`

Expand source code

    from dataclasses import dataclass
    from typing import NamedTuple, Tuple

    from scipy.spatial import ConvexHull # pylint: disable=no-name-in-module
    import cv2
    import numpy as np
    from mlworkflow import AugmentedDataset

    from deepsport_utilities.calib import Point3D, Point2D, Calib
    from deepsport_utilities.court import Court, court_dim
    from deepsport_utilities.utils import BoundingBox, crop_padded

    from deepsport_utilities.ds.instants_dataset import InstantsDataset, Instant, InstantKey, DownloadFlags

    class ViewKey(NamedTuple):
        instant_key: InstantKey
        camera: int
        index: int

        @property
        def arena_label(self):
            return self.instant_key.arena_label

        @property
        def timestamp(self):
            return self.instant_key.timestamp

        @property
        def game_id(self):
            return self.instant_key.game_id

    class View():
        def __init__(self, all_images, calib, annotations=None, **kwargs):
            self.image = all_images[0]
            self.all_images = all_images
            self.calib = calib
            self.annotations = annotations
            for k, v in kwargs.items():
                setattr(self, k, v)

        def draw(self):
            image = self.image.copy()

            coord_2D_int = lambda x: self.calib.project_3D_to_2D(x).to_int_tuple()

            for annotation in self.annotations:
                if annotation.type == "player":
                    head = annotation.head
                    hips = annotation.hips
                    foot1 = annotation.foot1
                    foot2 = annotation.foot2

                    color = [0, 0, 0]
                    color[annotation.team-1] = 255
                    cv2.line(image, coord_2D_int(head), coord_2D_int(hips), color, 3)
                    cv2.line(image, coord_2D_int(hips), coord_2D_int(foot1), color, 3)
                    cv2.line(image, coord_2D_int(hips), coord_2D_int(foot2), color, 3)

                elif annotation.type == "ball":
                    ball =  annotation.center
                    color = [255, 255, 0]
                    cv2.circle(image, coord_2D_int(ball), 5, color, -1)

            return image

    class ViewDescription():
        def __init__(self, camera, index, box: BoundingBox, **kwargs):
            self.camera = camera
            self.index = index
            self.box = box
            self.data = kwargs

    @dataclass
    class ViewBuilder():
        """ Margin given in world coordinates by default, except if `margin_in_pixels` is True.
        """
        margin: float = 0
        padding: int = None
        margin_in_pixels: bool = False
        def __post_init__(self):
            if self.margin_in_pixels:
                self.padding = self.margin if self.padding is None else self.padding
                self.compute_margin = lambda calib, p3d: self.margin
            else:
                self.padding = self.padding or 0
                self.compute_margin = lambda calib, p3d: calib.compute_length2D(p3d, self.margin)

        def compute_box(self, point3D_list: list, calib: Calib):
            point_and_margin_2D = [(
                    calib.project_3D_to_2D(point3D),
                    self.compute_margin(calib, point3D)
                ) for point3D in point3D_list]
            left_idx  = np.argmin([d[0].x for d in point_and_margin_2D])
            right_idx = np.argmax([d[0].x for d in point_and_margin_2D])
            top_idx   = np.argmin([d[0].y for d in point_and_margin_2D])
            bot_idx   = np.argmax([d[0].y for d in point_and_margin_2D])

            left  = max(-self.padding,             point_and_margin_2D[left_idx][0].x - point_and_margin_2D[left_idx][1])
            right = min(calib.width+self.padding,  point_and_margin_2D[right_idx][0].x + point_and_margin_2D[right_idx][1])
            top   = max(-self.padding,             point_and_margin_2D[top_idx][0].y - point_and_margin_2D[top_idx][1])
            bot   = min(calib.height+self.padding, point_and_margin_2D[bot_idx][0].y + point_and_margin_2D[bot_idx][1])

            return BoundingBox(float(left), float(top), float(right-left), float(bot-top))

    class BuildCameraViews(ViewBuilder):
        """ Builds a view for each camera (margin parameter is useless)
        """
        def __call__(self, instant_key: InstantKey, instant:Instant):
            for c in range(instant.num_cameras):
                yield ViewDescription(c, 0, BoundingBox(0, 0, instant.calibs[c].width, instant.calibs[c].height), court_dim=instant.court_dim)

    @dataclass
    class BuildCourtViews(ViewBuilder):
        """ Builds a view including all the court keypoints visible on each camera
            Note: keypoints are duplicated at 2m from the floor
        """
        height: float = 300
        def __call__(self, instant_key: InstantKey, instant:Instant):
            for c in range(instant.num_cameras):
                calib = instant.calibs[c]
                visible_edges = Court(instant.rule_type).visible_edges(calib)
                court_keypoints = []
                for p1, p2 in visible_edges:
                    court_keypoints = court_keypoints + [p1, p1+Point3D(0,0,-self.height), p2, p2+Point3D(0,0,-self.height)]
                yield ViewDescription(c, 0, self.compute_box(court_keypoints, calib))

    @dataclass
    class BuildPlayersViews(ViewBuilder):
        """ Builds a view around the players visible on each camera
            min_annotations: minimum required number of person to use that camera
        """
        min_annotations: int = 1
        def __call__(self, instant_key: InstantKey, instant: Instant):
            for c in range(instant.num_cameras):
                annotations = [a for a in instant.annotations if a.camera == c and a.type == "player" and a.team > 0]
                if len(annotations) < self.min_annotations:
                    continue

                keypoints = []
                for a in annotations:
                    keypoints += [a.head, a.hips, a.foot1, a.foot2]

                yield ViewDescription(c, 0, self.compute_box(keypoints, instant.calibs[c]))

    @dataclass
    class BuildThumbnailViews(ViewBuilder):
        """ Builds a view around each person (players, referee)
        """
        with_annotations: bool = True
        with_detections: bool = False
        with_random: Tuple[int, bool] = 0
        with_occlusions: bool = False
        BODY_HEIGHT = 180
        threshold = 0.25
        def __post_init__(self):
            self.with_random = 10 if isinstance(self.with_random, bool) and self.with_random else self.with_random

        def check_density_map(self, density_map, box, threshold):
            if 0 in density_map[box.y_slice, box.x_slice].shape:
                return False
            if np.mean(density_map[box.y_slice, box.x_slice]) <= threshold:
                return True
            return False

        def sample_density_map(self, density_map):
            # avoid going too close to other detections
            dilation = cv2.dilate(density_map, np.ones((3,3)), iterations=10)
            indices_y, indices_x = np.where(dilation == 0)
            # random choic a position in the image
            i = np.random.randint(0, len(indices_x))
            return np.array([[indices_x[i], indices_y[i]]])

        @staticmethod
        def fill_density_map(density_map, box):
            density_map[box.y_slice, box.x_slice] += 1

        def __call__(self, instant_key: InstantKey, instant:Instant):
            # Set random seed with timestamp
            random_state = np.random.get_state()
            np.random.seed(instant_key.timestamp & 0xFFFFFFFF)

            instant.density_maps = [np.zeros(img.shape[0:2], dtype=np.uint8) for img in instant.images]

            for c in range(instant.num_cameras):
                calib = instant.calibs[c]
                density_map = instant.density_maps[c]
                index = 0

                # From annotation
                for a in [a for a in instant.annotations if a.type == "player" and calib.projects_in(a.hips) and self.with_annotations]:
                    keypoints = [a.head, a.hips, a.foot1, a.foot2]
                    box = self.compute_box(keypoints, calib)
                    #if self.check_density_map(instant.density_maps, box, c, 1+self.threshold) or self.with_occlusions:
                    yield ViewDescription(c, index, box, origin='annotation', annotation=a, density_map=density_map)
                    self.fill_density_map(density_map, box)
                    index = index + 1

                if self.with_detections:
                    # From keemotion foregrounddetector detections
                    # FIXME:
                    for detection in []:#[d for d in instant.fg_detections if d.camera == c and calib.projects_in(d.feet) and self.with_detections]:
                        keypoints = [ detection.feet, detection.feet + Point3D(0, 0, -self.BODY_HEIGHT) ]
                        box = self.compute_box(keypoints, calib)
                        if self.check_density_map(density_map, box, self.threshold):
                            yield ViewDescription(c, index, box, origin='detection', detection=detection, density_map=density_map)
                            self.fill_density_map(density_map, box)
                            index = index + 1

                # From random
                if self.with_random:
                    raise NotImplementedError
                    # TODO: use Calib.visible_edge() to replace the function "find_court_intersection_with_camera_border"
                    court_keypoints_3D = []# find_court_intersection_with_camera_border(calib, instant.rule_type)
                    court_keypoints_2D = np.array([calib.project_3D_to_2D(p).to_list() for p in court_keypoints_3D])
                    convex_hull = ConvexHull(court_keypoints_2D)
                    points = np.array([court_keypoints_2D[i,:] for i in convex_hull.vertices]).astype(np.int32)
                    court = np.ones(instant.images[c].shape[0:2], dtype=np.uint8)
                    density_map[cv2.fillPoly(court, [points], 0)==1] += 1

                for _ in range(self.with_random):
                    feet = calib.project_2D_to_3D(Point2D(self.sample_density_map(density_map)), Z=0)
                    keypoints = [feet, feet + Point3D(0, 0, -self.BODY_HEIGHT)]
                    box = self.compute_box(keypoints, calib)
                    if self.check_density_map(density_map, box, self.threshold):
                        yield ViewDescription(c, index, box, origin='random', density_map=density_map)
                        self.fill_density_map(density_map, box)
                        index = index + 1

            # Restore random seed
            np.random.set_state(random_state)

    class BuildBallViews(ViewBuilder):
        def __init__(self, *args, origins=["annotation"], **kwargs):
            super().__init__(*args, **kwargs)
            self.origins = origins
        def __call__(self, instant_key: InstantKey, instant: Instant):
            balls = [a for a in instant.annotations + getattr(instant, "detections", []) if a.type == 'ball']
            predicate = lambda a: a.origin in self.origins
            for idx, ball in enumerate(filter(predicate, balls)):
                keypoints = [ball.center]
                c = int(ball.camera)
                timestamp = getattr(instant, "timestamps", [instant.timestamp]*instant.num_cameras)[c] # timestamps may be different for each camera
                yield ViewDescription(c, idx, self.compute_box(keypoints, instant.calibs[c]), ball=ball, timestamp=timestamp)

    class BuildHeadsViews(ViewBuilder):
        def __call__(self, instant_key: InstantKey, instant: Instant):
            for idx, player in enumerate([a for a in instant.annotations if a.type == "player"]):
                c = int(player.camera)
                keypoints = [player.head]
                yield ViewDescription(c, idx, self.compute_box(keypoints, instant.calibs[c]), annotation=player)

    class ViewsDataset(AugmentedDataset):
        """ Dataset of views built according the given ViewBuilder, extracted from the given InstantDataset.
            Arguments:
                instants_dataset   - the InstantDataset from which views are built
                view_builder       - the ViewBuilder dictating what type of view is to be created
                output_shape       - view aspect ratio (or exact dimension if 'rescale' is given)
                rescale            - tells whether the view should be rescaled to output_shape size
                crop               - tells whether the original image should be cropped or a rectangle
                                     should be drawn instead (for debug purposes)
        """
        def __init__(self, instants_dataset: InstantsDataset, view_builder: ViewBuilder, output_shape=None, rescale=True, crop=True):
            super().__init__(instants_dataset)
            self.view_builder = view_builder
            self.output_shape = output_shape
            self.rescale = rescale
            self.crop = crop

        def _crop_view(self, view_description: ViewDescription, instant: Instant, **kwargs):
            padding = self.view_builder.padding
            c = view_description.camera
            input_height, input_width, _ = instant.images[c].shape
            aspect_ratio = self.output_shape[0]/self.output_shape[1] if self.output_shape else None
            x_slice, y_slice = view_description.box.increase_box(
                max_width=input_width,
                max_height=input_height,
                aspect_ratio=aspect_ratio,
                padding=self.view_builder.padding
            )
            all_images = []

            if self.crop:
                for offset in instant.offsets:
                    index = (c,offset)
                    if index not in instant.all_images:
                        continue # that offset was not downloaded with the download flag of instants dataset
                    image = crop_padded(instant.all_images[index], x_slice, y_slice, padding)
                    if self.rescale and self.output_shape:
                        image = cv2.resize(image, self.output_shape)
                    all_images.append(image)
                calib = instant.calibs[c].crop(x_slice, y_slice) # handles negative `slice.start` positions
                if self.rescale and self.output_shape:
                    calib = calib.scale(*self.output_shape)
                if instant.download_flags & DownloadFlags.WITH_HUMAN_SEGMENTATION_MASKS and instant.human_masks:
                    human_masks = crop_padded(instant.human_masks[c], x_slice, y_slice, padding)
                    if self.rescale and self.output_shape:
                        human_masks = cv2.resize(human_masks, self.output_shape)
                else:
                    human_masks = None
            else:
                for offset in instant.offsets:
                    # the coordinates of the rectangle below are probably wrong see documentation of cv2.rectangle
                    raise NotImplementedError("TODO: check rectangle coordinates")
                    image = cv2.rectangle(instant.all_images[(c, offset)], (x_slice.start, x_slice.stop), (y_slice.start, y_slice.stop), (255,0,0), 10)
                    all_images.append(image)
                calib = instant.calibs[c]
                human_masks = instant.human_masks[c]

            return View(all_images, calib, instant.annotations, rule_type=instant.rule_type, human_masks=human_masks, **kwargs)

        def augment(self, instant_key: InstantKey, instant: Instant):
            random_state = np.random.get_state()
            np.random.seed(instant_key.timestamp & 0xFFFFFFFF)

            for view_description in self.view_builder(instant_key, instant):
                view_key = ViewKey(instant_key, view_description.camera, view_description.index)
                yield view_key, self._crop_view(view_description, instant, **view_description.data)

            # required to keep the random seed random
            np.random.set_state(random_state)

    # TODO: delete this 2 functions and use the Court or Calib objects
    def get_court_keypoints(rule_type):
        w, h = court_dim[rule_type]
        return [Point3D(0,0,0), Point3D(w,0,0), Point3D(0,h,0), Point3D(w,h,0)]
    def find_court_intersection_with_camera_border(calib, rule_type):
        def dichotomy(inside, outside, max_it=10):
            middle = Point3D((inside+outside)/2)
            if max_it == 0:
                return middle
            max_it = max_it - 1
            return dichotomy(middle, outside, max_it) if calib.projects_in(middle) else dichotomy(inside, middle, max_it)
        def find_point_inside(p1, p2, max_it=5):
            assert not calib.projects_in(p1) and not calib.projects_in(p2)
            middle = Point3D((p1+p2)/2)
            if calib.projects_in(middle):
                return middle
            if max_it == 0:
                return None
            point_inside = find_point_inside(middle, p2, max_it-1)
            if point_inside is not None:
                return point_inside
            return find_point_inside(middle, p1, max_it-1)

        points_inside = list()
        court_keypoints = get_court_keypoints(rule_type)

        for p1, p2 in zip(court_keypoints, court_keypoints[1:] + [court_keypoints[0]]):
            if calib.projects_in(p1) and calib.projects_in(p2):
                points_inside.append(p1)
                points_inside.append(p2)
            elif calib.projects_in(p1):
                points_inside.append(p1)
                points_inside.append(dichotomy(p1, p2))
            elif calib.projects_in(p2):
                points_inside.append(p2)
                points_inside.append(dichotomy(p2, p1))
            else:
                point_inside = find_point_inside(p1, p2)
                if point_inside is not None:
                    points_inside.append(dichotomy(point_inside, p2))
                    points_inside.append(dichotomy(point_inside, p1))

        return points_inside

## Functions

`def find_court_intersection_with_camera_border(calib, rule_type)`
  Expand source code

      def find_court_intersection_with_camera_border(calib, rule_type):
          def dichotomy(inside, outside, max_it=10):
              middle = Point3D((inside+outside)/2)
              if max_it == 0:
                  return middle
              max_it = max_it - 1
              return dichotomy(middle, outside, max_it) if calib.projects_in(middle) else dichotomy(inside, middle, max_it)
          def find_point_inside(p1, p2, max_it=5):
              assert not calib.projects_in(p1) and not calib.projects_in(p2)
              middle = Point3D((p1+p2)/2)
              if calib.projects_in(middle):
                  return middle
              if max_it == 0:
                  return None
              point_inside = find_point_inside(middle, p2, max_it-1)
              if point_inside is not None:
                  return point_inside
              return find_point_inside(middle, p1, max_it-1)

          points_inside = list()
          court_keypoints = get_court_keypoints(rule_type)

          for p1, p2 in zip(court_keypoints, court_keypoints[1:] + [court_keypoints[0]]):
              if calib.projects_in(p1) and calib.projects_in(p2):
                  points_inside.append(p1)
                  points_inside.append(p2)
              elif calib.projects_in(p1):
                  points_inside.append(p1)
                  points_inside.append(dichotomy(p1, p2))
              elif calib.projects_in(p2):
                  points_inside.append(p2)
                  points_inside.append(dichotomy(p2, p1))
              else:
                  point_inside = find_point_inside(p1, p2)
                  if point_inside is not None:
                      points_inside.append(dichotomy(point_inside, p2))
                      points_inside.append(dichotomy(point_inside, p1))

          return points_inside

`def get_court_keypoints(rule_type)`
  Expand source code

      def get_court_keypoints(rule_type):
          w, h = court_dim[rule_type]
          return [Point3D(0,0,0), Point3D(w,0,0), Point3D(0,h,0), Point3D(w,h,0)]

## Classes

`class BuildBallViews (*args, origins=['annotation'], **kwargs)`

  Margin given in world coordinates by default, except if `margin_in_pixels` is True.

  Expand source code

      class BuildBallViews(ViewBuilder):
          def __init__(self, *args, origins=["annotation"], **kwargs):
              super().__init__(*args, **kwargs)
              self.origins = origins
          def __call__(self, instant_key: InstantKey, instant: Instant):
              balls = [a for a in instant.annotations + getattr(instant, "detections", []) if a.type == 'ball']
              predicate = lambda a: a.origin in self.origins
              for idx, ball in enumerate(filter(predicate, balls)):
                  keypoints = [ball.center]
                  c = int(ball.camera)
                  timestamp = getattr(instant, "timestamps", [instant.timestamp]*instant.num_cameras)[c] # timestamps may be different for each camera
                  yield ViewDescription(c, idx, self.compute_box(keypoints, instant.calibs[c]), ball=ball, timestamp=timestamp)

  ### Ancestors

  - ViewBuilder

  ### Class variables

  `var margin : float`
  `var margin_in_pixels : bool`
  `var padding : int`
`class BuildCameraViews (margin: float = 0, padding: int = None, margin_in_pixels: bool = False)`

  Builds a view for each camera (margin parameter is useless)

  Expand source code

      class BuildCameraViews(ViewBuilder):
          """ Builds a view for each camera (margin parameter is useless)
          """
          def __call__(self, instant_key: InstantKey, instant:Instant):
              for c in range(instant.num_cameras):
                  yield ViewDescription(c, 0, BoundingBox(0, 0, instant.calibs[c].width, instant.calibs[c].height), court_dim=instant.court_dim)

  ### Ancestors

  - ViewBuilder

  ### Class variables

  `var margin : float`
  `var margin_in_pixels : bool`
  `var padding : int`
`class BuildCourtViews (margin: float = 0, padding: int = None, margin_in_pixels: bool = False, height: float = 300)`

  Builds a view including all the court keypoints visible on each camera Note: keypoints are duplicated at 2m from the floor

  Expand source code

      class BuildCourtViews(ViewBuilder):
          """ Builds a view including all the court keypoints visible on each camera
              Note: keypoints are duplicated at 2m from the floor
          """
          height: float = 300
          def __call__(self, instant_key: InstantKey, instant:Instant):
              for c in range(instant.num_cameras):
                  calib = instant.calibs[c]
                  visible_edges = Court(instant.rule_type).visible_edges(calib)
                  court_keypoints = []
                  for p1, p2 in visible_edges:
                      court_keypoints = court_keypoints + [p1, p1+Point3D(0,0,-self.height), p2, p2+Point3D(0,0,-self.height)]
                  yield ViewDescription(c, 0, self.compute_box(court_keypoints, calib))

  ### Ancestors

  - ViewBuilder

  ### Class variables

  `var height : float`
`class BuildHeadsViews (margin: float = 0, padding: int = None, margin_in_pixels: bool = False)`

  Margin given in world coordinates by default, except if `margin_in_pixels` is True.

  Expand source code

      class BuildHeadsViews(ViewBuilder):
          def __call__(self, instant_key: InstantKey, instant: Instant):
              for idx, player in enumerate([a for a in instant.annotations if a.type == "player"]):
                  c = int(player.camera)
                  keypoints = [player.head]
                  yield ViewDescription(c, idx, self.compute_box(keypoints, instant.calibs[c]), annotation=player)

  ### Ancestors

  - ViewBuilder

  ### Class variables

  `var margin : float`
  `var margin_in_pixels : bool`
  `var padding : int`
`class BuildPlayersViews (margin: float = 0, padding: int = None, margin_in_pixels: bool = False, min_annotations: int = 1)`

  Builds a view around the players visible on each camera min_annotations: minimum required number of person to use that camera

  Expand source code

      class BuildPlayersViews(ViewBuilder):
          """ Builds a view around the players visible on each camera
              min_annotations: minimum required number of person to use that camera
          """
          min_annotations: int = 1
          def __call__(self, instant_key: InstantKey, instant: Instant):
              for c in range(instant.num_cameras):
                  annotations = [a for a in instant.annotations if a.camera == c and a.type == "player" and a.team > 0]
                  if len(annotations) < self.min_annotations:
                      continue

                  keypoints = []
                  for a in annotations:
                      keypoints += [a.head, a.hips, a.foot1, a.foot2]

                  yield ViewDescription(c, 0, self.compute_box(keypoints, instant.calibs[c]))

  ### Ancestors

  - ViewBuilder

  ### Class variables

  `var min_annotations : int`
`class BuildThumbnailViews (margin: float = 0, padding: int = None, margin_in_pixels: bool = False, with_annotations: bool = True, with_detections: bool = False, with_random: Tuple[int, bool] = 0, with_occlusions: bool = False)`

  Builds a view around each person (players, referee)

  Expand source code

      class BuildThumbnailViews(ViewBuilder):
          """ Builds a view around each person (players, referee)
          """
          with_annotations: bool = True
          with_detections: bool = False
          with_random: Tuple[int, bool] = 0
          with_occlusions: bool = False
          BODY_HEIGHT = 180
          threshold = 0.25
          def __post_init__(self):
              self.with_random = 10 if isinstance(self.with_random, bool) and self.with_random else self.with_random

          def check_density_map(self, density_map, box, threshold):
              if 0 in density_map[box.y_slice, box.x_slice].shape:
                  return False
              if np.mean(density_map[box.y_slice, box.x_slice]) <= threshold:
                  return True
              return False

          def sample_density_map(self, density_map):
              # avoid going too close to other detections
              dilation = cv2.dilate(density_map, np.ones((3,3)), iterations=10)
              indices_y, indices_x = np.where(dilation == 0)
              # random choic a position in the image
              i = np.random.randint(0, len(indices_x))
              return np.array([[indices_x[i], indices_y[i]]])

          @staticmethod
          def fill_density_map(density_map, box):
              density_map[box.y_slice, box.x_slice] += 1

          def __call__(self, instant_key: InstantKey, instant:Instant):
              # Set random seed with timestamp
              random_state = np.random.get_state()
              np.random.seed(instant_key.timestamp & 0xFFFFFFFF)

              instant.density_maps = [np.zeros(img.shape[0:2], dtype=np.uint8) for img in instant.images]

              for c in range(instant.num_cameras):
                  calib = instant.calibs[c]
                  density_map = instant.density_maps[c]
                  index = 0

                  # From annotation
                  for a in [a for a in instant.annotations if a.type == "player" and calib.projects_in(a.hips) and self.with_annotations]:
                      keypoints = [a.head, a.hips, a.foot1, a.foot2]
                      box = self.compute_box(keypoints, calib)
                      #if self.check_density_map(instant.density_maps, box, c, 1+self.threshold) or self.with_occlusions:
                      yield ViewDescription(c, index, box, origin='annotation', annotation=a, density_map=density_map)
                      self.fill_density_map(density_map, box)
                      index = index + 1

                  if self.with_detections:
                      # From keemotion foregrounddetector detections
                      # FIXME:
                      for detection in []:#[d for d in instant.fg_detections if d.camera == c and calib.projects_in(d.feet) and self.with_detections]:
                          keypoints = [ detection.feet, detection.feet + Point3D(0, 0, -self.BODY_HEIGHT) ]
                          box = self.compute_box(keypoints, calib)
                          if self.check_density_map(density_map, box, self.threshold):
                              yield ViewDescription(c, index, box, origin='detection', detection=detection, density_map=density_map)
                              self.fill_density_map(density_map, box)
                              index = index + 1

                  # From random
                  if self.with_random:
                      raise NotImplementedError
                      # TODO: use Calib.visible_edge() to replace the function "find_court_intersection_with_camera_border"
                      court_keypoints_3D = []# find_court_intersection_with_camera_border(calib, instant.rule_type)
                      court_keypoints_2D = np.array([calib.project_3D_to_2D(p).to_list() for p in court_keypoints_3D])
                      convex_hull = ConvexHull(court_keypoints_2D)
                      points = np.array([court_keypoints_2D[i,:] for i in convex_hull.vertices]).astype(np.int32)
                      court = np.ones(instant.images[c].shape[0:2], dtype=np.uint8)
                      density_map[cv2.fillPoly(court, [points], 0)==1] += 1

                  for _ in range(self.with_random):
                      feet = calib.project_2D_to_3D(Point2D(self.sample_density_map(density_map)), Z=0)
                      keypoints = [feet, feet + Point3D(0, 0, -self.BODY_HEIGHT)]
                      box = self.compute_box(keypoints, calib)
                      if self.check_density_map(density_map, box, self.threshold):
                          yield ViewDescription(c, index, box, origin='random', density_map=density_map)
                          self.fill_density_map(density_map, box)
                          index = index + 1

              # Restore random seed
              np.random.set_state(random_state)

  ### Ancestors

  - ViewBuilder

  ### Class variables

  `var BODY_HEIGHT`
  `var threshold`
  `var with_annotations : bool`
  `var with_detections : bool`
  `var with_occlusions : bool`
  `var with_random : Tuple[int, bool]`

  ### Static methods

  `def fill_density_map(density_map, box)`
    Expand source code

        @staticmethod
        def fill_density_map(density_map, box):
            density_map[box.y_slice, box.x_slice] += 1

  ### Methods

  `def check_density_map(self, density_map, box, threshold)`
    Expand source code

        def check_density_map(self, density_map, box, threshold):
            if 0 in density_map[box.y_slice, box.x_slice].shape:
                return False
            if np.mean(density_map[box.y_slice, box.x_slice]) <= threshold:
                return True
            return False

  `def sample_density_map(self, density_map)`
    Expand source code

        def sample_density_map(self, density_map):
            # avoid going too close to other detections
            dilation = cv2.dilate(density_map, np.ones((3,3)), iterations=10)
            indices_y, indices_x = np.where(dilation == 0)
            # random choic a position in the image
            i = np.random.randint(0, len(indices_x))
            return np.array([[indices_x[i], indices_y[i]]])

`class View (all_images, calib, annotations=None, **kwargs)`
  Expand source code

      class View():
          def __init__(self, all_images, calib, annotations=None, **kwargs):
              self.image = all_images[0]
              self.all_images = all_images
              self.calib = calib
              self.annotations = annotations
              for k, v in kwargs.items():
                  setattr(self, k, v)

          def draw(self):
              image = self.image.copy()

              coord_2D_int = lambda x: self.calib.project_3D_to_2D(x).to_int_tuple()

              for annotation in self.annotations:
                  if annotation.type == "player":
                      head = annotation.head
                      hips = annotation.hips
                      foot1 = annotation.foot1
                      foot2 = annotation.foot2

                      color = [0, 0, 0]
                      color[annotation.team-1] = 255
                      cv2.line(image, coord_2D_int(head), coord_2D_int(hips), color, 3)
                      cv2.line(image, coord_2D_int(hips), coord_2D_int(foot1), color, 3)
                      cv2.line(image, coord_2D_int(hips), coord_2D_int(foot2), color, 3)

                  elif annotation.type == "ball":
                      ball =  annotation.center
                      color = [255, 255, 0]
                      cv2.circle(image, coord_2D_int(ball), 5, color, -1)

              return image

  ### Methods

  `def draw(self)`
    Expand source code

        def draw(self):
            image = self.image.copy()

            coord_2D_int = lambda x: self.calib.project_3D_to_2D(x).to_int_tuple()

            for annotation in self.annotations:
                if annotation.type == "player":
                    head = annotation.head
                    hips = annotation.hips
                    foot1 = annotation.foot1
                    foot2 = annotation.foot2

                    color = [0, 0, 0]
                    color[annotation.team-1] = 255
                    cv2.line(image, coord_2D_int(head), coord_2D_int(hips), color, 3)
                    cv2.line(image, coord_2D_int(hips), coord_2D_int(foot1), color, 3)
                    cv2.line(image, coord_2D_int(hips), coord_2D_int(foot2), color, 3)

                elif annotation.type == "ball":
                    ball =  annotation.center
                    color = [255, 255, 0]
                    cv2.circle(image, coord_2D_int(ball), 5, color, -1)

            return image

`class ViewBuilder (margin: float = 0, padding: int = None, margin_in_pixels: bool = False)`

  Margin given in world coordinates by default, except if `margin_in_pixels` is True.

  Expand source code

      class ViewBuilder():
          """ Margin given in world coordinates by default, except if `margin_in_pixels` is True.
          """
          margin: float = 0
          padding: int = None
          margin_in_pixels: bool = False
          def __post_init__(self):
              if self.margin_in_pixels:
                  self.padding = self.margin if self.padding is None else self.padding
                  self.compute_margin = lambda calib, p3d: self.margin
              else:
                  self.padding = self.padding or 0
                  self.compute_margin = lambda calib, p3d: calib.compute_length2D(p3d, self.margin)

          def compute_box(self, point3D_list: list, calib: Calib):
              point_and_margin_2D = [(
                      calib.project_3D_to_2D(point3D),
                      self.compute_margin(calib, point3D)
                  ) for point3D in point3D_list]
              left_idx  = np.argmin([d[0].x for d in point_and_margin_2D])
              right_idx = np.argmax([d[0].x for d in point_and_margin_2D])
              top_idx   = np.argmin([d[0].y for d in point_and_margin_2D])
              bot_idx   = np.argmax([d[0].y for d in point_and_margin_2D])

              left  = max(-self.padding,             point_and_margin_2D[left_idx][0].x - point_and_margin_2D[left_idx][1])
              right = min(calib.width+self.padding,  point_and_margin_2D[right_idx][0].x + point_and_margin_2D[right_idx][1])
              top   = max(-self.padding,             point_and_margin_2D[top_idx][0].y - point_and_margin_2D[top_idx][1])
              bot   = min(calib.height+self.padding, point_and_margin_2D[bot_idx][0].y + point_and_margin_2D[bot_idx][1])

              return BoundingBox(float(left), float(top), float(right-left), float(bot-top))

  ### Subclasses

  - BuildBallViews
  - BuildCameraViews
  - BuildCourtViews
  - BuildHeadsViews
  - BuildPlayersViews
  - BuildThumbnailViews

  ### Class variables

  `var margin : float`
  `var margin_in_pixels : bool`
  `var padding : int`

  ### Methods

  `def compute_box(self, point3D_list: list, calib: Calib)`
    Expand source code

        def compute_box(self, point3D_list: list, calib: Calib):
            point_and_margin_2D = [(
                    calib.project_3D_to_2D(point3D),
                    self.compute_margin(calib, point3D)
                ) for point3D in point3D_list]
            left_idx  = np.argmin([d[0].x for d in point_and_margin_2D])
            right_idx = np.argmax([d[0].x for d in point_and_margin_2D])
            top_idx   = np.argmin([d[0].y for d in point_and_margin_2D])
            bot_idx   = np.argmax([d[0].y for d in point_and_margin_2D])

            left  = max(-self.padding,             point_and_margin_2D[left_idx][0].x - point_and_margin_2D[left_idx][1])
            right = min(calib.width+self.padding,  point_and_margin_2D[right_idx][0].x + point_and_margin_2D[right_idx][1])
            top   = max(-self.padding,             point_and_margin_2D[top_idx][0].y - point_and_margin_2D[top_idx][1])
            bot   = min(calib.height+self.padding, point_and_margin_2D[bot_idx][0].y + point_and_margin_2D[bot_idx][1])

            return BoundingBox(float(left), float(top), float(right-left), float(bot-top))

`class ViewDescription (camera, index, box: BoundingBox, **kwargs)`
  Expand source code

      class ViewDescription():
          def __init__(self, camera, index, box: BoundingBox, **kwargs):
              self.camera = camera
              self.index = index
              self.box = box
              self.data = kwargs

`class ViewKey (instant_key: InstantKey, camera: int, index: int)`

  ViewKey(instant_key, camera, index)

  Expand source code

      class ViewKey(NamedTuple):
          instant_key: InstantKey
          camera: int
          index: int

          @property
          def arena_label(self):
              return self.instant_key.arena_label

          @property
          def timestamp(self):
              return self.instant_key.timestamp

          @property
          def game_id(self):
              return self.instant_key.game_id

  ### Ancestors

  - builtins.tuple

  ### Instance variables

  `var arena_label`
    Expand source code

        @property
        def arena_label(self):
            return self.instant_key.arena_label

  `var camera : int`

    Alias for field number 1

  `var game_id`
    Expand source code

        @property
        def game_id(self):
            return self.instant_key.game_id

  `var index : int`

    Alias for field number 2

  `var instant_key : InstantKey`

    Alias for field number 0

  `var timestamp`
    Expand source code

        @property
        def timestamp(self):
            return self.instant_key.timestamp

`class ViewsDataset (instants_dataset: InstantsDataset, view_builder: ViewBuilder, output_shape=None, rescale=True, crop=True)`

  Dataset of views built according the given ViewBuilder, extracted from the given InstantDataset.

  ## Arguments

  instants_dataset - the InstantDataset from which views are built view_builder - the ViewBuilder dictating what type of view is to be created output_shape - view aspect ratio (or exact dimension if 'rescale' is given) rescale - tells whether the view should be rescaled to output_shape size crop - tells whether the original image should be cropped or a rectangle should be drawn instead (for debug purposes)

  Expand source code

      class ViewsDataset(AugmentedDataset):
          """ Dataset of views built according the given ViewBuilder, extracted from the given InstantDataset.
              Arguments:
                  instants_dataset   - the InstantDataset from which views are built
                  view_builder       - the ViewBuilder dictating what type of view is to be created
                  output_shape       - view aspect ratio (or exact dimension if 'rescale' is given)
                  rescale            - tells whether the view should be rescaled to output_shape size
                  crop               - tells whether the original image should be cropped or a rectangle
                                       should be drawn instead (for debug purposes)
          """
          def __init__(self, instants_dataset: InstantsDataset, view_builder: ViewBuilder, output_shape=None, rescale=True, crop=True):
              super().__init__(instants_dataset)
              self.view_builder = view_builder
              self.output_shape = output_shape
              self.rescale = rescale
              self.crop = crop

          def _crop_view(self, view_description: ViewDescription, instant: Instant, **kwargs):
              padding = self.view_builder.padding
              c = view_description.camera
              input_height, input_width, _ = instant.images[c].shape
              aspect_ratio = self.output_shape[0]/self.output_shape[1] if self.output_shape else None
              x_slice, y_slice = view_description.box.increase_box(
                  max_width=input_width,
                  max_height=input_height,
                  aspect_ratio=aspect_ratio,
                  padding=self.view_builder.padding
              )
              all_images = []

              if self.crop:
                  for offset in instant.offsets:
                      index = (c,offset)
                      if index not in instant.all_images:
                          continue # that offset was not downloaded with the download flag of instants dataset
                      image = crop_padded(instant.all_images[index], x_slice, y_slice, padding)
                      if self.rescale and self.output_shape:
                          image = cv2.resize(image, self.output_shape)
                      all_images.append(image)
                  calib = instant.calibs[c].crop(x_slice, y_slice) # handles negative `slice.start` positions
                  if self.rescale and self.output_shape:
                      calib = calib.scale(*self.output_shape)
                  if instant.download_flags & DownloadFlags.WITH_HUMAN_SEGMENTATION_MASKS and instant.human_masks:
                      human_masks = crop_padded(instant.human_masks[c], x_slice, y_slice, padding)
                      if self.rescale and self.output_shape:
                          human_masks = cv2.resize(human_masks, self.output_shape)
                  else:
                      human_masks = None
              else:
                  for offset in instant.offsets:
                      # the coordinates of the rectangle below are probably wrong see documentation of cv2.rectangle
                      raise NotImplementedError("TODO: check rectangle coordinates")
                      image = cv2.rectangle(instant.all_images[(c, offset)], (x_slice.start, x_slice.stop), (y_slice.start, y_slice.stop), (255,0,0), 10)
                      all_images.append(image)
                  calib = instant.calibs[c]
                  human_masks = instant.human_masks[c]

              return View(all_images, calib, instant.annotations, rule_type=instant.rule_type, human_masks=human_masks, **kwargs)

          def augment(self, instant_key: InstantKey, instant: Instant):
              random_state = np.random.get_state()
              np.random.seed(instant_key.timestamp & 0xFFFFFFFF)

              for view_description in self.view_builder(instant_key, instant):
                  view_key = ViewKey(instant_key, view_description.camera, view_description.index)
                  yield view_key, self._crop_view(view_description, instant, **view_description.data)

              # required to keep the random seed random
              np.random.set_state(random_state)

  ### Ancestors

  - mlworkflow.datasets.AugmentedDataset
  - mlworkflow.datasets.Dataset

  ### Methods

  `def augment(self, instant_key: InstantKey, instant: Instant)`
    Expand source code

        def augment(self, instant_key: InstantKey, instant: Instant):
            random_state = np.random.get_state()
            np.random.seed(instant_key.timestamp & 0xFFFFFFFF)

            for view_description in self.view_builder(instant_key, instant):
                view_key = ViewKey(instant_key, view_description.camera, view_description.index)
                yield view_key, self._crop_view(view_description, instant, **view_description.data)

            # required to keep the random seed random
            np.random.set_state(random_state)