title: Android camera2 output to ImageReader format YUV_420_888 still slow
body: <p>I'm trying to get the Android camera2 running in the background service, then process the frame in the callback ImageReader.OnImageAvailableListener. I already use the suggested raw format YUV_420_888 to get max fps, however I only get around 7fps on the resolution 640x480. This is even slower than what I get using the old Camera interface( I want to upgrade to Camera2 to get higher fps ) or with the OpenCV JavaCameraView( I can't use this because I need to run processing in the background service ). </p>
  
  <p>Below is my service class. What am I missing?</p>
  
  <p>My phone is Redmi Note 3 running Android 5.0.2</p>
  
  <pre><code>public class Camera2ServiceYUV extends Service {
      protected static final String TAG = "VideoProcessing";
      protected static final int CAMERACHOICE = CameraCharacteristics.LENS_FACING_BACK;
      protected CameraDevice cameraDevice;
      protected CameraCaptureSession captureSession;
      protected ImageReader imageReader;
  
      // A semaphore to prevent the app from exiting before closing the camera.
      private Semaphore mCameraOpenCloseLock = new Semaphore(1);
  
  
      public static final String RESULT_RECEIVER = "resultReceiver";
      private static final int JPEG_COMPRESSION = 90;
  
      public static final int RESULT_OK = 0;
      public static final int RESULT_DEVICE_NO_CAMERA= 1;
      public static final int RESULT_GET_CAMERA_FAILED = 2;
      public static final int RESULT_ALREADY_RUNNING = 3;
      public static final int RESULT_NOT_RUNNING = 4;
  
      private static final String START_SERVICE_COMMAND = "startServiceCommands";
      private static final int COMMAND_NONE = -1;
      private static final int COMMAND_START = 0;
      private static final int COMMAND_STOP = 1;
  
      private boolean mRunning = false;
      public Camera2ServiceYUV() {
      }
  
      public static void startToStart(Context context, ResultReceiver resultReceiver) {
          Intent intent = new Intent(context, Camera2ServiceYUV.class);
          intent.putExtra(START_SERVICE_COMMAND, COMMAND_START);
          intent.putExtra(RESULT_RECEIVER, resultReceiver);
          context.startService(intent);
      }
  
      public static void startToStop(Context context, ResultReceiver resultReceiver) {
          Intent intent = new Intent(context, Camera2ServiceYUV.class);
          intent.putExtra(START_SERVICE_COMMAND, COMMAND_STOP);
          intent.putExtra(RESULT_RECEIVER, resultReceiver);
          context.startService(intent);
      }
  
      // SERVICE INTERFACE
      @Override
      public int onStartCommand(Intent intent, int flags, int startId) {
          switch (intent.getIntExtra(START_SERVICE_COMMAND, COMMAND_NONE)) {
              case COMMAND_START:
                  startCamera(intent);
                  break;
              case COMMAND_STOP:
                  stopCamera(intent);
                  break;
              default:
                  throw new UnsupportedOperationException("Cannot start the camera service with an illegal command.");
          }
  
          return START_STICKY;
  
  
  
      }
  
      @Override
      public void onDestroy() {
          try {
              captureSession.abortCaptures();
          } catch (CameraAccessException e) {
              Log.e(TAG, e.getMessage());
          }
          captureSession.close();
      }
  
      @Override
      public IBinder onBind(Intent intent) {
          return null;
      }
  
  
      // CAMERA2 INTERFACE
      /**
       * 1. The android CameraManager class is used to manage all the camera devices in our android device
       * Each camera device has a range of properties and settings that describe the device.
       * It can be obtained through the camera characteristics.
       */
      public void startCamera(Intent intent) {
  
          final ResultReceiver resultReceiver = intent.getParcelableExtra(RESULT_RECEIVER);
  
          if (mRunning) {
              resultReceiver.send(RESULT_ALREADY_RUNNING, null);
              return;
          }
          mRunning = true;
  
          CameraManager manager = (CameraManager) getSystemService(CAMERA_SERVICE);
          try {
              if (!mCameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) {
                  throw new RuntimeException("Time out waiting to lock camera opening.");
              }
              String pickedCamera = getCamera(manager);
              Log.e(TAG,"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + pickedCamera);
              manager.openCamera(pickedCamera, cameraStateCallback, null);
              CameraCharacteristics characteristics = manager.getCameraCharacteristics(pickedCamera);
              Size[] jpegSizes = null;
              if (characteristics != null) {
                  jpegSizes = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP).getOutputSizes(ImageFormat.YUV_420_888);
              }
              int width = 640;
              int height = 480;
  //            if (jpegSizes != null &amp;&amp; 0 &lt; jpegSizes.length) {
  //                width = jpegSizes[jpegSizes.length -1].getWidth();
  //                height = jpegSizes[jpegSizes.length - 1].getHeight();
  //            }
  //            for(Size s : jpegSizes)
  //            {
  //                Log.e(TAG,"Size = " + s.toString());
  //            }
  
  
              // DEBUG
              StreamConfigurationMap map = characteristics.get(
                      CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
              if (map == null) {
                  return;
              }
              Log.e(TAG,"Width = " + width + ", Height = " + height);
              Log.e(TAG,"output stall duration = " + map.getOutputStallDuration(ImageFormat.YUV_420_888, new Size(width,height)) );
              Log.e(TAG,"Min output stall duration = " + map.getOutputMinFrameDuration(ImageFormat.YUV_420_888, new Size(width,height)) );
  
  //            Size[] sizeList = map.getInputSizes(ImageFormat.YUV_420_888);
  //            for(Size s : sizeList)
  //            {
  //                Log.e(TAG,"Size = " + s.toString());
  //            }
  
              imageReader = ImageReader.newInstance(width, height, ImageFormat.YUV_420_888, 2 /* images buffered */);
              imageReader.setOnImageAvailableListener(onImageAvailableListener, null);
              Log.i(TAG, "imageReader created");
          } catch (CameraAccessException e) {
              Log.e(TAG, e.getMessage());
              resultReceiver.send(RESULT_DEVICE_NO_CAMERA, null);
          }catch (InterruptedException e) {
              resultReceiver.send(RESULT_GET_CAMERA_FAILED, null);
              throw new RuntimeException("Interrupted while trying to lock camera opening.", e);
          }
          catch(SecurityException se)
          {
              resultReceiver.send(RESULT_GET_CAMERA_FAILED, null);
              throw new RuntimeException("Security permission exception while trying to open the camera.", se);
          }
  
          resultReceiver.send(RESULT_OK, null);
      }
  
      // We can pick the camera being used, i.e. rear camera in this case.
      private String getCamera(CameraManager manager) {
          try {
              for (String cameraId : manager.getCameraIdList()) {
                  CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId);
                  int cOrientation = characteristics.get(CameraCharacteristics.LENS_FACING);
                  if (cOrientation == CAMERACHOICE) {
                      return cameraId;
                  }
              }
          } catch (CameraAccessException e) {
              e.printStackTrace();
          }
          return null;
      }
  
  
      /**
       * 1.1 Callbacks when the camera changes its state - opened, disconnected, or error.
       */
      protected CameraDevice.StateCallback cameraStateCallback = new CameraDevice.StateCallback() {
          @Override
          public void onOpened(@NonNull CameraDevice camera) {
              Log.i(TAG, "CameraDevice.StateCallback onOpened");
              mCameraOpenCloseLock.release();
              cameraDevice = camera;
              createCaptureSession();
          }
  
          @Override
          public void onDisconnected(@NonNull CameraDevice camera) {
              Log.w(TAG, "CameraDevice.StateCallback onDisconnected");
              mCameraOpenCloseLock.release();
              camera.close();
              cameraDevice = null;
          }
  
          @Override
          public void onError(@NonNull CameraDevice camera, int error) {
              Log.e(TAG, "CameraDevice.StateCallback onError " + error);
              mCameraOpenCloseLock.release();
              camera.close();
              cameraDevice = null;
          }
      };
  
  
      /**
       * 2. To capture or stream images from a camera device, the application must first create
       * a camera capture captureSession.
       * The camera capture needs a surface to output what has been captured, in this case
       * we use ImageReader in order to access the frame data.
       */
      public void createCaptureSession() {
          try {
              cameraDevice.createCaptureSession(Arrays.asList(imageReader.getSurface()), sessionStateCallback, null);
          } catch (CameraAccessException e) {
              Log.e(TAG, e.getMessage());
          }
      }
  
          protected CameraCaptureSession.StateCallback sessionStateCallback = new CameraCaptureSession.StateCallback() {
          @Override
          public void onConfigured(@NonNull CameraCaptureSession session) {
              Log.i(TAG, "CameraCaptureSession.StateCallback onConfigured");
  
              // The camera is already closed
              if (null == cameraDevice) {
                  return;
              }
  
              // When the captureSession is ready, we start to grab the frame.
              Camera2ServiceYUV.this.captureSession = session;
  
              try {
                  session.setRepeatingRequest(createCaptureRequest(), null, null);
              } catch (CameraAccessException e) {
                  Log.e(TAG, e.getMessage());
              }
          }
  
          @Override
          public void onConfigureFailed(@NonNull CameraCaptureSession session) {
              Log.e(TAG, "CameraCaptureSession.StateCallback onConfigureFailed");
          }
      };
  
      /**
       * 3. The application then needs to construct a CaptureRequest, which defines all the capture parameters
       *    needed by a camera device to capture a single image.
       */
      private CaptureRequest createCaptureRequest() {
          try {
              /**
               * Check other templates for further details.
               * TEMPLATE_MANUAL = 6
               * TEMPLATE_PREVIEW = 1
               * TEMPLATE_RECORD = 3
               * TEMPLATE_STILL_CAPTURE = 2
               * TEMPLATE_VIDEO_SNAPSHOT = 4
               * TEMPLATE_ZERO_SHUTTER_LAG = 5
               *
               * TODO: can set camera features like auto focus, auto flash here
               * captureRequestBuilder.set(CaptureRequest.CONTROL_AF_MODE,CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
               */
              CaptureRequest.Builder captureRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
  //            captureRequestBuilder.set(CaptureRequest.EDGE_MODE,
  //                    CaptureRequest.EDGE_MODE_OFF);
  //            captureRequestBuilder.set(
  //                    CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE,
  //                    CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE_ON);
  //            captureRequestBuilder.set(
  //                    CaptureRequest.COLOR_CORRECTION_ABERRATION_MODE,
  //                    CaptureRequest.COLOR_CORRECTION_ABERRATION_MODE_OFF);
  //            captureRequestBuilder.set(CaptureRequest.NOISE_REDUCTION_MODE,
  //                    CaptureRequest.NOISE_REDUCTION_MODE_OFF);
  //            captureRequestBuilder.set(CaptureRequest.CONTROL_AF_TRIGGER,
  //                    CaptureRequest.CONTROL_AF_TRIGGER_CANCEL);
  //
  //            captureRequestBuilder.set(CaptureRequest.CONTROL_AE_LOCK, true);
  //            captureRequestBuilder.set(CaptureRequest.CONTROL_AWB_LOCK, true);
  
              captureRequestBuilder.addTarget(imageReader.getSurface());
              return captureRequestBuilder.build();
          } catch (CameraAccessException e) {
              Log.e(TAG, e.getMessage());
              return null;
          }
      }
  
  
      /**
       * ImageReader provides a surface for the camera to output what has been captured.
       * Upon the image available, call processImage() to process the image as desired.
       */
      private long frameTime = 0;
      private ImageReader.OnImageAvailableListener onImageAvailableListener = new ImageReader.OnImageAvailableListener() {
          @Override
          public void onImageAvailable(ImageReader reader) {
              Log.i(TAG, "called ImageReader.OnImageAvailable");
              Image img = reader.acquireLatestImage();
              if (img != null) {
                  if( frameTime != 0 )
                  {
                      Log.e(TAG, "fps = " + (float)(1000.0 / (float)(SystemClock.elapsedRealtime() - frameTime)) + " fps");
                  }
                  frameTime = SystemClock.elapsedRealtime();
                  img.close();
              }
          }
      };
  
      private void processImage(Image image) {
          Mat outputImage = imageToMat(image);
          Bitmap bmp = Bitmap.createBitmap(outputImage.cols(), outputImage.rows(), Bitmap.Config.ARGB_8888);
          Utils.bitmapToMat(bmp, outputImage);
          Point mid = new Point(0, 0);
          Point inEnd = new Point(outputImage.cols(), outputImage.rows());
          Imgproc.line(outputImage, mid, inEnd, new Scalar(255, 0, 0), 2, Core.LINE_AA, 0);
          Utils.matToBitmap(outputImage, bmp);
  
          Intent broadcast = new Intent();
          broadcast.setAction("your_load_photo_action");
          broadcast.putExtra("BitmapImage", bmp);
          sendBroadcast(broadcast);
      }
  
      private Mat imageToMat(Image image) {
          ByteBuffer buffer;
          int rowStride;
          int pixelStride;
          int width = image.getWidth();
          int height = image.getHeight();
          int offset = 0;
  
          Image.Plane[] planes = image.getPlanes();
          byte[] data = new byte[image.getWidth() * image.getHeight() * ImageFormat.getBitsPerPixel(ImageFormat.YUV_420_888) / 8];
          byte[] rowData = new byte[planes[0].getRowStride()];
  
          for (int i = 0; i &lt; planes.length; i++) {
              buffer = planes[i].getBuffer();
              rowStride = planes[i].getRowStride();
              pixelStride = planes[i].getPixelStride();
              int w = (i == 0) ? width : width / 2;
              int h = (i == 0) ? height : height / 2;
              for (int row = 0; row &lt; h; row++) {
                  int bytesPerPixel = ImageFormat.getBitsPerPixel(ImageFormat.YUV_420_888) / 8;
                  if (pixelStride == bytesPerPixel) {
                      int length = w * bytesPerPixel;
                      buffer.get(data, offset, length);
  
                      // Advance buffer the remainder of the row stride, unless on the last row.
                      // Otherwise, this will throw an IllegalArgumentException because the buffer
                      // doesn't include the last padding.
                      if (h - row != 1) {
                          buffer.position(buffer.position() + rowStride - length);
                      }
                      offset += length;
                  } else {
  
                      // On the last row only read the width of the image minus the pixel stride
                      // plus one. Otherwise, this will throw a BufferUnderflowException because the
                      // buffer doesn't include the last padding.
                      if (h - row == 1) {
                          buffer.get(rowData, 0, width - pixelStride + 1);
                      } else {
                          buffer.get(rowData, 0, rowStride);
                      }
  
                      for (int col = 0; col &lt; w; col++) {
                          data[offset++] = rowData[col * pixelStride];
                      }
                  }
              }
          }
  
          // Finally, create the Mat.
          Mat mat = new Mat(height + height / 2, width, CV_8UC1);
          mat.put(0, 0, data);
  
          return mat;
      }
  
  
      private void stopCamera(Intent intent) {
          ResultReceiver resultReceiver = intent.getParcelableExtra(RESULT_RECEIVER);
  
          if (!mRunning) {
              resultReceiver.send(RESULT_NOT_RUNNING, null);
              return;
          }
  
          closeCamera();
  
          resultReceiver.send(RESULT_OK, null);
  
          mRunning = false;
          Log.d(TAG, "Service is finished.");
      }
  
      /**
       * Closes the current {@link CameraDevice}.
       */
      private void closeCamera() {
          try {
              mCameraOpenCloseLock.acquire();
              if (null != captureSession) {
                  captureSession.close();
                  captureSession = null;
              }
              if (null != cameraDevice) {
                  cameraDevice.close();
                  cameraDevice = null;
              }
              if (null != imageReader) {
                  imageReader.close();
                  imageReader = null;
              }
          } catch (InterruptedException e) {
              throw new RuntimeException("Interrupted while trying to lock camera closing.", e);
          } finally {
              mCameraOpenCloseLock.release();
          }
      }
  }
  </code></pre>