Merge "Camera: ITS: Allow spurious faces detected by opencv" into android14-tests-dev

Author: Treehugger Robot

apps/CameraITS/tests/scene2_a/test_num_faces.py

@@ -32,95 +32,14 @@
 _CV2_FACE_SCALE_FACTOR = 1.05  # 5% step for resizing image to find face
 _CV2_FACE_MIN_NEIGHBORS = 4  # recommended 3-6: higher for less faces
 _CV2_GREEN = (0, 1, 0)
-_CV2_RED = (1, 0, 0)
-_FACE_CENTER_MATCH_TOL_X = 10  # 10 pixels or ~1.5% in 640x480 image
-_FACE_CENTER_MATCH_TOL_Y = 20  # 20 pixels or ~4% in 640x480 image
-_FACE_CENTER_MIN_LOGGING_DIST = 50
 _FD_MODE_OFF, _FD_MODE_SIMPLE, _FD_MODE_FULL = 0, 1, 2
-_MIN_NUM_FACES_ALIGNED = 2
-_MIN_CENTER_DELTA = 15
 _NAME = os.path.splitext(os.path.basename(__file__))[0]
 _NUM_FACES = 3
 _NUM_TEST_FRAMES = 20
 _TEST_REQUIRED_MPC = 34
 _W, _H = 640, 480
 
 
-def eliminate_duplicate_centers(coordinates_list):
-  """Checks center coordinates of OpenCV's face rectangles
-
-  Method makes sure that the list of face rectangles' centers do not
-  contain duplicates from the same face.
-
-  Args:
-    coordinates_list: list; coordinates of face rectangles' centers
-  Returns:
-    non_duplicate_list: list; coordinates of face rectangles' centers
-    without duplicates on the same face
-  """
-  output = set()
-
-  for i, xy1 in enumerate(coordinates_list):
-    for j, xy2 in enumerate(coordinates_list):
-      if distance.euclidean(xy1, xy2) < _MIN_CENTER_DELTA:
-        continue
-      if xy1 not in output:
-        output.add(xy1)
-      else:
-        output.add(xy2)
-  return list(output)
-
-
-def match_face_locations(faces_cropped, faces_opencv, mode, img, img_name):
-  """Assert face locations between two methods.
-
-  Method determines if center of opencv face boxes is within face detection
-  face boxes. Using math.hypot to measure the distance between the centers,
-  as math.dist is not available for python versions before 3.8.
-
-  Args:
-    faces_cropped: list of lists with (l, r, t, b) for each face.
-    faces_opencv: list of lists with (x, y, w, h) for each face.
-    mode: int indicating face detection mode
-    img: np image array
-    img_name: text string with path to image file
-  """
-  # turn faces_opencv into list of center locations
-  faces_opencv_center = [(x+w//2, y+h//2) for (x, y, w, h) in faces_opencv]
-  cropped_faces_centers = [
-      ((l+r)//2, (t+b)//2) for (l, r, t, b) in faces_cropped]
-  faces_opencv_center.sort(key=lambda t: [t[1], t[0]])
-  cropped_faces_centers.sort(key=lambda t: [t[1], t[0]])
-  logging.debug('cropped face centers: %s', str(cropped_faces_centers))
-  logging.debug('opencv face center: %s', str(faces_opencv_center))
-  faces_opencv_centers = []
-  num_centers_aligned = 0
-
-  # eliminate duplicate openCV face rectangles' centers the same face
-  faces_opencv_centers = eliminate_duplicate_centers(faces_opencv_center)
-  logging.debug('opencv face centers: %s', str(faces_opencv_centers))
-
-  for (x, y) in faces_opencv_centers:
-    for (x1, y1) in cropped_faces_centers:
-      centers_dist = math.hypot(x-x1, y-y1)
-      if centers_dist < _FACE_CENTER_MIN_LOGGING_DIST:
-        logging.debug('centers_dist: %.3f', centers_dist)
-      if (abs(x-x1) < _FACE_CENTER_MATCH_TOL_X and
-          abs(y-y1) < _FACE_CENTER_MATCH_TOL_Y):
-        num_centers_aligned += 1
-
-  # If test failed, save image with green AND OpenCV red rectangles
-  image_processing_utils.write_image(img, img_name)
-  if num_centers_aligned < _MIN_NUM_FACES_ALIGNED:
-    for (x, y, w, h) in faces_opencv:
-      cv2.rectangle(img, (x, y), (x+w, y+h), _CV2_RED, 2)
-      image_processing_utils.write_image(img, img_name)
-      logging.debug('centered: %s', str(num_centers_aligned))
-    raise AssertionError(f'Mode {mode} face rectangles in wrong location(s)!. '
-                         f'Found {num_centers_aligned} rectangles near cropped '
-                         f'face centers, expected {_MIN_NUM_FACES_ALIGNED}')
-
-
 def check_face_bounding_box(rect, aw, ah, index):
   """Checks face bounding box is within the active array area.
 
@@ -187,32 +106,6 @@ def check_face_landmarks(face, fd_mode, index):
     raise AssertionError(f'Unknown face detection mode: {fd_mode}.')
 
 
-def correct_faces_for_crop(faces, img, crop):
-  """Correct face rectangles for sensor crop.
-
-  Args:
-    faces: list of dicts with face information
-    img: np image array
-    crop: dict of crop region size with 'top, right, left, bottom' as keys
-  Returns:
-    list of face locations (left, right, top, bottom) corrected
-  """
-  faces_corrected = []
-  cw, ch = crop['right'] - crop['left'], crop['bottom'] - crop['top']
-  logging.debug('crop region: %s', str(crop))
-  w = img.shape[1]
-  h = img.shape[0]
-  for rect in [face['bounds'] for face in faces]:
-    logging.debug('rect: %s', str(rect))
-    left = int(round((rect['left'] - crop['left']) * w / cw))
-    right = int(round((rect['right'] - crop['left']) * w / cw))
-    top = int(round((rect['top'] - crop['top']) * h / ch))
-    bottom = int(round((rect['bottom'] - crop['top']) * h / ch))
-    faces_corrected.append([left, right, top, bottom])
-  logging.debug('faces_corrected: %s', str(faces_corrected))
-  return faces_corrected
-
-
 class NumFacesTest(its_base_test.ItsBaseTest):
   """Test face detection with different skin tones.
   """
@@ -281,7 +174,8 @@ def test_num_faces(self):
 
             # draw boxes around faces in green
             crop_region = cap['metadata']['android.scaler.cropRegion']
-            faces_cropped = correct_faces_for_crop(faces, img, crop_region)
+            faces_cropped = opencv_processing_utils.correct_faces_for_crop(
+                faces, img, crop_region)
             for (l, r, t, b) in faces_cropped:
               cv2.rectangle(img, (l, t), (r, b), _CV2_GREEN, 2)
 
@@ -315,8 +209,8 @@ def test_num_faces(self):
               faces_opencv = opencv_processing_utils.find_opencv_faces(
                   img, _CV2_FACE_SCALE_FACTOR, _CV2_FACE_MIN_NEIGHBORS)
               if fd_mode:  # non-zero value for ON
-                match_face_locations(faces_cropped, faces_opencv,
-                                     fd_mode, img, img_name)
+                opencv_processing_utils.match_face_locations(
+                    faces_cropped, faces_opencv, img, img_name)
 
           if not faces:
             continue

apps/CameraITS/tests/scene2_d/test_autoframing.py

@@ -30,6 +30,7 @@
 _AUTOFRAMING_CONVERGED = 2
 _CV2_FACE_SCALE_FACTOR = 1.05  # 5% step for resizing image to find face
 _CV2_FACE_MIN_NEIGHBORS = 4  # recommended 3-6: higher for less faces
+_NAME = os.path.splitext(os.path.basename(__file__))[0]
 _NUM_TEST_FRAMES = 150
 _NUM_FACES = 3
 _W, _H = 640, 480
@@ -84,21 +85,32 @@ def test_autoframing(self):
         # Face detection and autoframing could take several frames to warm up,
         # but should detect the correct number of faces before the last frame
         if autoframing_state == _AUTOFRAMING_CONVERGED:
+          # Save image when autoframing state converges
+          control_zoom_ratio = cap['metadata']['android.control.zoomRatio']
+          logging.debug('Control zoom ratio: %d', control_zoom_ratio)
+          img = image_processing_utils.convert_capture_to_rgb_image(
+              cap, props=props)
+          file_name_stem = os.path.join(self.log_path, _NAME)
+          img_name = f'{file_name_stem}.jpg'
+
+          # Save images with green boxes around faces
+          crop_region = cap['metadata']['android.scaler.cropRegion']
+          faces_cropped = opencv_processing_utils.correct_faces_for_crop(
+              faces, img, crop_region)
+          opencv_processing_utils.draw_green_boxes_around_faces(
+              img, faces_cropped, img_name)
+
           num_faces_found = len(faces)
           if num_faces_found != _NUM_FACES:
             raise AssertionError('Wrong num of faces found! Found: '
                                  f'{num_faces_found}, expected: {_NUM_FACES}')
 
           # Also check the faces with open cv to make sure the scene is not
           # distorted or anything.
-          img = image_processing_utils.convert_capture_to_rgb_image(
-              cap, props=props)
           opencv_faces = opencv_processing_utils.find_opencv_faces(
               img, _CV2_FACE_SCALE_FACTOR, _CV2_FACE_MIN_NEIGHBORS)
-          num_opencv_faces = len(opencv_faces)
-          if num_opencv_faces != _NUM_FACES:
-            raise AssertionError('Wrong num of faces found with OpenCV! Found: '
-                                 f'{num_opencv_faces}, expected: {_NUM_FACES}')
+          opencv_processing_utils.match_face_locations(
+              faces_cropped, opencv_faces, img, img_name)
           break
 
         # Autoframing didn't converge till the last frame

apps/CameraITS/utils/opencv_processing_utils.py

@@ -21,6 +21,8 @@
 import cv2
 import numpy
 
+from scipy.spatial import distance
+
 import capture_request_utils
 import error_util
 import image_processing_utils
@@ -50,13 +52,20 @@
 
 CV2_LINE_THICKNESS = 3  # line thickness for drawing on images
 CV2_RED = (255, 0, 0)  # color in cv2 to draw lines
+CV2_GREEN = (0, 1, 0)
 CV2_THRESHOLD_BLOCK_SIZE = 11
 CV2_THRESHOLD_CONSTANT = 2
 
 CV2_HOME_DIRECTORY = os.path.dirname(cv2.__file__)
 CV2_ALTERNATE_DIRECTORY = pathlib.Path(CV2_HOME_DIRECTORY).parents[3]
 HAARCASCADE_FILE_NAME = 'haarcascade_frontalface_default.xml'
 
+FACES_ALIGNED_MIN_NUM = 2
+FACE_CENTER_MATCH_TOL_X = 10  # 10 pixels or ~1.5% in 640x480 image
+FACE_CENTER_MATCH_TOL_Y = 20  # 20 pixels or ~4% in 640x480 image
+FACE_CENTER_MIN_LOGGING_DIST = 50
+FACE_MIN_CENTER_DELTA = 15
+
 FOV_THRESH_TELE25 = 25
 FOV_THRESH_TELE40 = 40
 FOV_THRESH_TELE = 60
@@ -907,3 +916,120 @@ def get_angle(input_img):
     return None
 
   return numpy.median(filtered_angles)
+
+
+def correct_faces_for_crop(faces, img, crop):
+  """Correct face rectangles for sensor crop.
+
+  Args:
+    faces: list of dicts with face information
+    img: np image array
+    crop: dict of crop region size with 'top, right, left, bottom' as keys
+  Returns:
+    list of face locations (left, right, top, bottom) corrected
+  """
+  faces_corrected = []
+  cw, ch = crop['right'] - crop['left'], crop['bottom'] - crop['top']
+  logging.debug('crop region: %s', str(crop))
+  w = img.shape[1]
+  h = img.shape[0]
+  for rect in [face['bounds'] for face in faces]:
+    logging.debug('rect: %s', str(rect))
+    left = int(round((rect['left'] - crop['left']) * w / cw))
+    right = int(round((rect['right'] - crop['left']) * w / cw))
+    top = int(round((rect['top'] - crop['top']) * h / ch))
+    bottom = int(round((rect['bottom'] - crop['top']) * h / ch))
+    faces_corrected.append([left, right, top, bottom])
+  logging.debug('faces_corrected: %s', str(faces_corrected))
+  return faces_corrected
+
+
+def eliminate_duplicate_centers(coordinates_list):
+  """Checks center coordinates of OpenCV's face rectangles
+
+  Method makes sure that the list of face rectangles' centers do not
+  contain duplicates from the same face
+
+  Args:
+    coordinates_list: list; coordinates of face rectangles' centers
+  Returns:
+    non_duplicate_list: list; coordinates of face rectangles' centers
+    without duplicates on the same face
+  """
+  output = set()
+
+  for i, xy1 in enumerate(coordinates_list):
+    for j, xy2 in enumerate(coordinates_list):
+      if distance.euclidean(xy1, xy2) < FACE_MIN_CENTER_DELTA:
+        continue
+      if xy1 not in output:
+        output.add(xy1)
+      else:
+        output.add(xy2)
+  return list(output)
+
+
+def match_face_locations(faces_cropped, faces_opencv, img, img_name):
+  """Assert face locations between two methods.
+
+  Method determines if center of opencv face boxes is within face detection
+  face boxes. Using math.hypot to measure the distance between the centers,
+  as math.dist is not available for python versions before 3.8.
+
+  Args:
+    faces_cropped: list of lists with (l, r, t, b) for each face.
+    faces_opencv: list of lists with (x, y, w, h) for each face.
+    img: numpy [0, 1] image array
+    img_name: text string with path to image file
+  """
+  # turn faces_opencv into list of center locations
+  faces_opencv_center = [(x+w//2, y+h//2) for (x, y, w, h) in faces_opencv]
+  cropped_faces_centers = [
+      ((l+r)//2, (t+b)//2) for (l, r, t, b) in faces_cropped]
+  faces_opencv_center.sort(key=lambda t: [t[1], t[0]])
+  cropped_faces_centers.sort(key=lambda t: [t[1], t[0]])
+  logging.debug('cropped face centers: %s', str(cropped_faces_centers))
+  logging.debug('opencv face center: %s', str(faces_opencv_center))
+  faces_opencv_centers = []
+  num_centers_aligned = 0
+
+  # eliminate duplicate openCV face rectangles' centers the same face
+  faces_opencv_centers = eliminate_duplicate_centers(faces_opencv_center)
+  logging.debug('opencv face centers: %s', str(faces_opencv_centers))
+
+  for (x, y) in faces_opencv_centers:
+    for (x1, y1) in cropped_faces_centers:
+      centers_dist = math.hypot(x-x1, y-y1)
+      if centers_dist < FACE_CENTER_MIN_LOGGING_DIST:
+        logging.debug('centers_dist: %.3f', centers_dist)
+      if (abs(x-x1) < FACE_CENTER_MATCH_TOL_X and
+          abs(y-y1) < FACE_CENTER_MATCH_TOL_Y):
+        num_centers_aligned += 1
+
+  # If test failed, save image with green AND OpenCV red rectangles
+  image_processing_utils.write_image(img, img_name)
+  if num_centers_aligned < FACES_ALIGNED_MIN_NUM:
+    for (x, y, w, h) in faces_opencv:
+      cv2.rectangle(img, (x, y), (x+w, y+h), tuple(numpy.array(CV2_RED)/255), 2)
+      image_processing_utils.write_image(img, img_name)
+      logging.debug('centered: %s', str(num_centers_aligned))
+    raise AssertionError(f'Face rectangles in wrong location(s)!. '
+                         f'Found {num_centers_aligned} rectangles near cropped '
+                         f'face centers, expected {FACES_ALIGNED_MIN_NUM}')
+
+
+def draw_green_boxes_around_faces(img, faces_cropped, img_name):
+  """Correct face rectangles for sensor crop.
+
+  Args:
+    img: numpy [0, 1] image array
+    faces_cropped: list of lists with (l, r, t, b) for each face
+    img_name: text string with path to image file
+  Returns:
+    image with green rectangles
+  """
+  # draw boxes around faces in green and save image
+  for (l, r, t, b) in faces_cropped:
+    cv2.rectangle(img, (l, t), (r, b), CV2_GREEN, 2)
+  image_processing_utils.write_image(img, img_name)
+