SnapdragonCamera:reduce memory cost

1.limit max preview size to 1080p
2.set max image count of ImageReader to 3.
3.Clear ZSL Queue when start recording.
4.use repeatingBurst instead of repeatingRequest.
5.set ZSL queue size to 3.
6.set ZSL snapshot to 15fps.
7.remove ZslFalloff queue.

Change-Id: I87ebec71d5b85b3e5f12a61c5898e43e6e184d61

5 files changed, 157 insertions, 94 deletions

diff --git a/src/com/android/camera/CaptureModule.java b/src/com/android/camera/CaptureModule.java
index ad17a7fbd..58d27d94b 100755
--- a/src/com/android/camera/CaptureModule.java
+++ b/src/com/android/camera/CaptureModule.java
@@ -634,7 +634,21 @@ public class CaptureModule implements CameraModule, PhotoController,
             }
             showBokehStatusMessage(id, result);
             processCaptureResult(result);
-            mPostProcessor.onMetaAvailable(result);
+            if (mPostProcessor.isZSLEnabled() && getCameraMode() != DUAL_MODE) {
+                boolean zsl = false;
+                List<CaptureResult> resultList = result.getPartialResults();
+                for (CaptureResult r : resultList) {
+                    if (r.getRequest().containsTarget(mImageReader[id].getSurface())) {
+                        zsl = true;
+                        break;
+                    }
+                }
+                if (zsl){
+                    mPostProcessor.onMetaAvailable(result);
+                }
+             } else {
+                mPostProcessor.onMetaAvailable(result);
+            }
         }
     };
 
@@ -1201,9 +1215,15 @@ public class CaptureModule implements CameraModule, PhotoController,
                                                 .build(), mCaptureCallback, mCameraHandler);
                                     }
                                 } else {
-                                    if (mCaptureSession[id] != null) {
-                                        mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
-                                                .build(), mCaptureCallback, mCameraHandler);
+                                    if (mPostProcessor.isZSLEnabled() && getCameraMode() !=
+                                            DUAL_MODE) {
+                                        setRepeatingBurstForZSL(id);
+                                    } else {
+                                        if (mCaptureSession[id] != null) {
+                                            mCaptureSession[id].setRepeatingRequest(
+                                                    mPreviewRequestBuilder[id].build(),
+                                                    mCaptureCallback, mCameraHandler);
+                                        }
                                     }
                                 }
 
@@ -1319,8 +1339,12 @@ public class CaptureModule implements CameraModule, PhotoController,
         applyAERegions(mPreviewRequestBuilder[id], id);
         mPreviewRequestBuilder[id].setTag(id);
         try {
-            mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
-                    .build(), mCaptureCallback, mCameraHandler);
+            if (mPostProcessor.isZSLEnabled() && getCameraMode() != DUAL_MODE) {
+                setRepeatingBurstForZSL(BAYER_ID);
+            } else {
+                mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
+                        .build(), mCaptureCallback, mCameraHandler);
+            }
         } catch (CameraAccessException | IllegalStateException e) {
             e.printStackTrace();
         }
@@ -1344,8 +1368,12 @@ public class CaptureModule implements CameraModule, PhotoController,
         applyAERegions(mPreviewRequestBuilder[id], id);
         mPreviewRequestBuilder[id].setTag(id);
         try {
-            mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
-                    .build(), mCaptureCallback, mCameraHandler);
+            if (mPostProcessor.isZSLEnabled() && getCameraMode() != DUAL_MODE) {
+                setRepeatingBurstForZSL(BAYER_ID);
+            } else {
+                mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
+                        .build(), mCaptureCallback, mCameraHandler);
+            }
         } catch (CameraAccessException | IllegalStateException e) {
             e.printStackTrace();
         }
@@ -1554,8 +1582,12 @@ public class CaptureModule implements CameraModule, PhotoController,
                 if (mLongshotActive && isFlashOn(id)) {
                     mCaptureSession[id].stopRepeating();
                     applyFlash(mPreviewRequestBuilder[id], id);
-                    mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
-                            .build(), mCaptureCallback, mCameraHandler);
+                    if (mPostProcessor.isZSLEnabled() && getCameraMode() != DUAL_MODE) {
+                        setRepeatingBurstForZSL(BAYER_ID);
+                    } else {
+                        mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
+                                .build(), mCaptureCallback, mCameraHandler);
+                    }
                 }
             }
         } catch (CameraAccessException e) {
@@ -1609,8 +1641,12 @@ public class CaptureModule implements CameraModule, PhotoController,
                 if (mLongshotActive && isFlashOn(id)) {
                     mCaptureSession[id].stopRepeating();
                     applyFlash(mPreviewRequestBuilder[id], id);
-                    mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
-                                            .build(), mCaptureCallback, mCameraHandler);
+                    if (mPostProcessor.isZSLEnabled() && getCameraMode() != DUAL_MODE) {
+                        setRepeatingBurstForZSL(id);
+                    } else {
+                        mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
+                                .build(), mCaptureCallback, mCameraHandler);
+                    }
                 }
             }
         } catch (CameraAccessException e) {
@@ -2336,8 +2372,12 @@ public class CaptureModule implements CameraModule, PhotoController,
         try {
             applySettingsForLockExposure(mPreviewRequestBuilder[id], id);
             mState[id] = STATE_WAITING_AE_LOCK;
-            mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id].build(),
-                    mCaptureCallback, mCameraHandler);
+            if (mPostProcessor.isZSLEnabled() && getCameraMode() != DUAL_MODE) {
+                setRepeatingBurstForZSL(id);
+            } else {
+                mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id].build(),
+                        mCaptureCallback, mCameraHandler);
+            }
         } catch (CameraAccessException | IllegalStateException e) {
             e.printStackTrace();
         }
@@ -3507,7 +3547,9 @@ public class CaptureModule implements CameraModule, PhotoController,
             mControlAFMode = CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE;
             closePreviewSession();
             mFrameProcessor.onClose();
-
+            if (mPostProcessor != null) {
+                mPostProcessor.enableZSLQueue(false);
+            }
             Size preview = mVideoPreviewSize;
             if (mHighSpeedCapture) {
                 preview = mVideoSize;
@@ -3538,7 +3580,6 @@ public class CaptureModule implements CameraModule, PhotoController,
             mFrameProcessor.setOutputSurface(surface);
             mFrameProcessor.setVideoOutputSurface(mMediaRecorder.getSurface());
             addPreviewSurface(mVideoRequestBuilder, surfaces, cameraId);
-
             if (mHighSpeedCapture)
                 mVideoRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, mHighSpeedFPSRange);
 
@@ -3553,6 +3594,7 @@ public class CaptureModule implements CameraModule, PhotoController,
                         CameraConstrainedHighSpeedCaptureSession session =
                                     (CameraConstrainedHighSpeedCaptureSession) mCurrentSession;
                         try {
+                            removeImageReaderSurfaces(mVideoRequestBuilder);
                             List list = session 
                                     .createHighSpeedRequestList(mVideoRequestBuilder.build());
                             session.setRepeatingBurst(list, mCaptureCallback, mCameraHandler);
@@ -3603,6 +3645,7 @@ public class CaptureModule implements CameraModule, PhotoController,
                         mCaptureSession[cameraId] = cameraCaptureSession;
                         try {
                             setUpVideoCaptureRequestBuilder(mVideoRequestBuilder, cameraId);
+                            removeImageReaderSurfaces(mVideoRequestBuilder);
                             mCurrentSession.setRepeatingRequest(mVideoRequestBuilder.build(),
                                     mCaptureCallback, mCameraHandler);
                         } catch (CameraAccessException e) {
@@ -3924,15 +3967,18 @@ public class CaptureModule implements CameraModule, PhotoController,
                 mUI.showRecordVideoForReview(thumbnail);
             }
         }
-
         if(mFrameProcessor != null) {
             mFrameProcessor.onOpen(getFrameProcFilterId(), mPreviewSize);
         }
+        if (mPostProcessor != null) {
+            mPostProcessor.enableZSLQueue(true);
+        }
         boolean changed = mUI.setPreviewSize(mPreviewSize.getWidth(), mPreviewSize.getHeight());
         if (changed) {
             mUI.hideSurfaceView();
             mUI.showSurfaceView();
         }
+
         createSessions();
         mUI.showUIafterRecording();
         mUI.resetTrackingFocus();
@@ -3955,6 +4001,14 @@ public class CaptureModule implements CameraModule, PhotoController,
         return dateFormat.format(date);
     }
 
+    private void removeImageReaderSurfaces(CaptureRequest.Builder builder) {
+        for (int i = 0; i < MAX_NUM_CAM; i++) {
+            if(mImageReader[i] != null){
+                builder.removeTarget(mImageReader[i].getSurface());
+            }
+        }
+    }
+
     private String generateVideoFilename(int outputFileFormat) {
         long dateTaken = System.currentTimeMillis();
         String title = createName(dateTaken);
@@ -4564,8 +4618,12 @@ public class CaptureModule implements CameraModule, PhotoController,
                     ((CameraConstrainedHighSpeedCaptureSession) session).setRepeatingBurst(list
                             , mCaptureCallback, mCameraHandler);
                 } else {
-                    mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
-                            .build(), mCaptureCallback, mCameraHandler);
+                    if (mPostProcessor.isZSLEnabled() && getCameraMode() != DUAL_MODE) {
+                        setRepeatingBurstForZSL(id);
+                    } else {
+                        mCaptureSession[id].setRepeatingRequest(mPreviewRequestBuilder[id]
+                                .build(), mCaptureCallback, mCameraHandler);
+                    }
                 }
 
             }
@@ -4969,8 +5027,12 @@ public class CaptureModule implements CameraModule, PhotoController,
             try {
                 if (checkSessionAndBuilder(mCaptureSession[BAYER_ID],
                         mPreviewRequestBuilder[BAYER_ID])) {
-                    mCaptureSession[BAYER_ID].setRepeatingRequest(mPreviewRequestBuilder[BAYER_ID]
-                            .build(), mCaptureCallback, mCameraHandler);
+                    if (mPostProcessor.isZSLEnabled() && getCameraMode() != DUAL_MODE) {
+                        setRepeatingBurstForZSL(BAYER_ID);
+                    } else {
+                        mCaptureSession[BAYER_ID].setRepeatingRequest(mPreviewRequestBuilder[BAYER_ID]
+                                .build(), mCaptureCallback, mCameraHandler);
+                    }
                 }
             } catch (CameraAccessException | IllegalStateException e) {
                 e.printStackTrace();
@@ -4996,8 +5058,12 @@ public class CaptureModule implements CameraModule, PhotoController,
             try {
                 if (checkSessionAndBuilder(mCaptureSession[FRONT_ID],
                         mPreviewRequestBuilder[FRONT_ID])) {
-                    mCaptureSession[FRONT_ID].setRepeatingRequest(mPreviewRequestBuilder[FRONT_ID]
-                            .build(), mCaptureCallback, mCameraHandler);
+                    if (mPostProcessor.isZSLEnabled()) {
+                        setRepeatingBurstForZSL(FRONT_ID);
+                    } else {
+                        mCaptureSession[FRONT_ID].setRepeatingRequest(mPreviewRequestBuilder[FRONT_ID]
+                                .build(), mCaptureCallback, mCameraHandler);
+                    }
                 }
             } catch (CameraAccessException | IllegalStateException e) {
                 e.printStackTrace();
@@ -5017,6 +5083,23 @@ public class CaptureModule implements CameraModule, PhotoController,
         }
     }
 
+    private void setRepeatingBurstForZSL(int id) throws CameraAccessException,IllegalStateException{
+        List<CaptureRequest> requests =
+                new ArrayList<CaptureRequest>();
+        CaptureRequest previewZslRequest = mPreviewRequestBuilder[id].build();
+        mPreviewRequestBuilder[id].removeTarget(mImageReader[id].getSurface());
+        CaptureRequest previewRequest = mPreviewRequestBuilder[id].build();
+        requests.add(previewZslRequest);
+        requests.add(previewRequest);
+        //restore the orginal request builder
+        mPreviewRequestBuilder[id].addTarget(mImageReader[id].getSurface());
+
+        if (mCaptureSession[id] != null) {
+            mCaptureSession[id].setRepeatingBurst(requests,
+                    mCaptureCallback,mCameraHandler);
+        }
+    }
+
     private boolean isPanoSetting(String value) {
         try {
             int mode = Integer.parseInt(value);
diff --git a/src/com/android/camera/imageprocessor/PostProcessor.java b/src/com/android/camera/imageprocessor/PostProcessor.java
index 8cec1b8a7..a8ed2dc8d 100755
--- a/src/com/android/camera/imageprocessor/PostProcessor.java
+++ b/src/com/android/camera/imageprocessor/PostProcessor.java
@@ -102,7 +102,7 @@ public class PostProcessor{
     public static final int FILTER_MAX = 8;
 
     //BestPicture requires 10 which is the biggest among filters
-    private static final int MAX_REQUIRED_IMAGE_NUM = 11;
+    private static final int MAX_REQUIRED_IMAGE_NUM = 3;
     private int mCurrentNumImage = 0;
     private ImageFilter mFilter;
     private int mFilterIndex;
@@ -134,6 +134,7 @@ public class PostProcessor{
     private ImageReader mImageReader;
     private ImageReader mZSLReprocessImageReader;
     private boolean mUseZSL = true;
+    private boolean mProcessZSL = true;
     private boolean mSaveRaw = false;
     private Handler mZSLHandler;
     private HandlerThread mZSLHandlerThread;
@@ -144,7 +145,6 @@ public class PostProcessor{
     private TotalCaptureResult mZSLFallOffResult = null;
     private boolean mIsZSLFallOff = false;
     private TotalCaptureResult mLatestResultForLongShot = null;
-    private LinkedList<ZSLQueue.ImageItem> mFallOffImages = new LinkedList<ZSLQueue.ImageItem>();
     private int mPendingContinuousRequestCount = 0;
     public int mMaxRequiredImageNum;
 
@@ -199,50 +199,6 @@ public class PostProcessor{
         }
     }
 
-    private void clearFallOffImage() {
-        for(ZSLQueue.ImageItem item: mFallOffImages ) {
-            try {
-                item.getImage().close();
-                Image raw = item.getRawImage();
-                if (raw != null) {
-                    raw.close();
-                }
-            } catch(Exception e) {
-            }
-        }
-        mFallOffImages.clear();
-    }
-
-    private ZSLQueue.ImageItem findFallOffImage(long timestamp) {
-        ZSLQueue.ImageItem foundImage = null;
-        for(ZSLQueue.ImageItem item: mFallOffImages ) {
-            if(item.getImage().getTimestamp() == timestamp) {
-                foundImage = item;
-                break;
-            }
-        }
-        if(foundImage != null) {
-            mFallOffImages.remove(foundImage);
-        }
-        return foundImage;
-    }
-
-    private void addFallOffImage(ZSLQueue.ImageItem item) {
-        mFallOffImages.add(item);
-        if(mFallOffImages.size() >= MAX_REQUIRED_IMAGE_NUM - 1) {
-            ZSLQueue.ImageItem it = mFallOffImages.getFirst();
-            try {
-                it.getImage().close();
-                Image raw = item.getRawImage();
-                if (raw != null) {
-                    raw.close();
-                }
-            } catch(Exception e) {
-            }
-            mFallOffImages.removeFirst();
-        }
-    }
-
     class ImageHandlerTask implements Runnable, ImageReader.OnImageAvailableListener {
         private ImageWrapper mImageWrapper = null;
         private ImageReader mRawImageReader = null;
@@ -250,6 +206,18 @@ public class PostProcessor{
 
         @Override
         public void onImageAvailable(ImageReader reader) {
+            if (!mProcessZSL) {
+                Image image = reader.acquireNextImage();
+                if (image != null) {
+                    image.close();
+                }
+                if (mSaveRaw && mRawImageReader != null) {
+                   Image rawImage = mRawImageReader.acquireNextImage();
+                   if (rawImage != null)
+                       rawImage.close();
+                }
+                return;
+            }
             try {
                 if(mUseZSL) {
                     if(mController.isLongShotActive() && mPendingContinuousRequestCount > 0) {
@@ -268,35 +236,18 @@ public class PostProcessor{
                         }
                         return;
                     }
+
                     if(mIsZSLFallOff) {
-                        Image image = reader.acquireNextImage();
-                        Image rawImage = null;
-                        if (mSaveRaw && mRawImageReader != null) {
-                            rawImage = mRawImageReader.acquireNextImage();
-                        }
-                        ZSLQueue.ImageItem imageItem = new ZSLQueue.ImageItem();
-                        imageItem.setImage(image,rawImage);
-                        if(mZSLFallOffResult == null) {
-                            addFallOffImage(imageItem);
-                            return;
-                        }
-                        addFallOffImage(imageItem);
-                        ZSLQueue.ImageItem foundImage = findFallOffImage(
-                                mZSLFallOffResult.get(CaptureResult.SENSOR_TIMESTAMP).longValue());
-                        if(foundImage != null && foundImage.getImage() != null) {
-                            Log.d(TAG,"ZSL fall off image is found");
-                            reprocessImage(foundImage.getImage(), mZSLFallOffResult);
-                            Image raw = foundImage.getRawImage();
+                        ZSLQueue.ImageItem foundImage = mZSLQueue.tryToGetMatchingItem();
+                        if (foundImage != null) {
+                            reprocessImage(foundImage.getImage(),foundImage.getMetadata());
+                            Image raw =  foundImage.getRawImage();
                             if (raw != null) {
                                 onRawImageToProcess(raw);
                             }
                             mIsZSLFallOff = false;
-                            clearFallOffImage();
                             mZSLFallOffResult = null;
-                        } else {
-                            clearFallOffImage();
                         }
-                        return;
                     }
 
                     Image image = reader.acquireLatestImage();
@@ -350,7 +301,6 @@ public class PostProcessor{
                 Log.e(TAG, "Max images has been already acquired. ");
                 mIsZSLFallOff = false;
                 mZSLFallOffResult = null;
-                clearFallOffImage();
             }
         }
 
@@ -456,12 +406,18 @@ public class PostProcessor{
         if(mController.getPreviewCaptureResult() == null ||
                 mController.getPreviewCaptureResult().get(CaptureResult.CONTROL_AE_STATE) == CameraMetadata.CONTROL_AE_STATE_FLASH_REQUIRED) {
             if(DEBUG_ZSL) Log.d(TAG, "Flash required image");
+            if (imageItem != null)
+                imageItem.closeImage();
             imageItem = null;
         }
         if (mController.isSelfieFlash()) {
+            if (imageItem != null)
+                imageItem.closeImage();
             imageItem = null;
         }
         if (mController.isLongShotActive()) {
+            if (imageItem != null)
+                imageItem.closeImage();
             imageItem = null;
         }
         if (imageItem != null) {
@@ -757,6 +713,17 @@ public class PostProcessor{
         mPendingContinuousRequestCount = 0;
     }
 
+    public void enableZSLQueue(boolean enable) {
+        if (enable) {
+            mProcessZSL = true;
+        } else {
+            mProcessZSL = false;
+            if (mZSLQueue != null) {
+                mZSLQueue.clear();
+            }
+        }
+    }
+
     private void startBackgroundThread() {
         mHandlerThread = new HandlerThread("PostProcessorThread");
         mHandlerThread.start();
diff --git a/src/com/android/camera/imageprocessor/ZSLQueue.java b/src/com/android/camera/imageprocessor/ZSLQueue.java
index 0509b8fb5..bf29267e3 100644..100755
--- a/src/com/android/camera/imageprocessor/ZSLQueue.java
+++ b/src/com/android/camera/imageprocessor/ZSLQueue.java
@@ -207,6 +207,19 @@ public class ZSLQueue {
         }
     }
 
+    public void clear() {
+        synchronized (mLock) {
+            for (int i = 0; i < mBuffer.length; i++) {
+                if (mBuffer[i] != null) {
+                    mBuffer[i].closeImage();
+                    mBuffer[i].closeMeta();
+                }
+            }
+            mImageHead = 0;
+            mMetaHead = 0;
+        }
+    }
+
     private boolean checkImageRequirement(TotalCaptureResult captureResult) {
         if( (captureResult.get(CaptureResult.LENS_STATE) != null &&
              captureResult.get(CaptureResult.LENS_STATE).intValue() == CaptureResult.LENS_STATE_MOVING)
diff --git a/src/com/android/camera/util/CameraUtil.java b/src/com/android/camera/util/CameraUtil.java
index 5af219c45..056dd0f68 100755
--- a/src/com/android/camera/util/CameraUtil.java
+++ b/src/com/android/camera/util/CameraUtil.java
@@ -576,7 +576,7 @@ public class CameraUtil {
         for (int i = 0; i < sizes.length; i++) {
             Point size = sizes[i];
             double ratio = (double) size.x / size.y;
-            if (Math.abs(ratio - targetRatio) > ASPECT_TOLERANCE) continue;
+            if (Math.abs(ratio - targetRatio) > ASPECT_TOLERANCE || size.x > 1080) continue;
 
             // Count sizes with height <= 1080p to mimic camera1 api behavior.
             if (size.y > MAX_ASPECT_HEIGHT) continue;
diff --git a/src/com/android/camera/util/PersistUtil.java b/src/com/android/camera/util/PersistUtil.java
index 9453ff803..a490cc751 100755
--- a/src/com/android/camera/util/PersistUtil.java
+++ b/src/com/android/camera/util/PersistUtil.java
@@ -76,7 +76,7 @@ public class PersistUtil {
             SystemProperties.getInt("persist.sys.camera.sensor.align",
                     CAMERA_SENSOR_HORIZONTAL_ALIGNED);
     private static final int CIRCULAR_BUFFER_SIZE_PERSIST =
-            SystemProperties.getInt("persist.sys.camera.zsl.buffer.size", 5);
+            SystemProperties.getInt("persist.sys.camera.zsl.buffer.size", 3);
     private static final int SAVE_TASK_MEMORY_LIMIT_IN_MB =
             SystemProperties.getInt("persist.sys.camera.perf.memlimit", 60);
     private static final boolean PERSIST_CAMERA_UI_AUTO_TEST_ENABLED =