[FFmpeg-cvslog] avcodec/videotoolboxenc: fix invalid session on iOS

[Thomas Guillem]

ffmpeg | branch: master | Thomas Guillem <thomas at gllm.fr> | Thu Jun 14 17:48:07 2018 +0200| [513e6a30fb013ca34812ccaaf3d090680ac868c5] | committer: Aman Gupta

avcodec/videotoolboxenc: fix invalid session on iOS

Cf. comment. Restart the VT session when the APP goes from foreground to
background and vice versa.

Signed-off-by: Aman Gupta <aman at tmm1.net>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=513e6a30fb013ca34812ccaaf3d090680ac868c5
---

 libavcodec/videotoolboxenc.c | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
index 1060055ba5..ac847358ab 100644
--- a/libavcodec/videotoolboxenc.c
+++ b/libavcodec/videotoolboxenc.c
@@ -2175,8 +2175,27 @@ static int create_cv_pixel_buffer(AVCodecContext   *avctx,
 #if TARGET_OS_IPHONE
     pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
     if (!pix_buf_pool) {
-        av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
-        return AVERROR_EXTERNAL;
+        /* On iOS, the VT session is invalidated when the APP switches from
+         * foreground to background and vice versa. Fetch the actual error code
+         * of the VT session to detect that case and restart the VT session
+         * accordingly. */
+        OSStatus vtstatus;
+
+        vtstatus = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
+        if (vtstatus == kVTInvalidSessionErr) {
+            CFRelease(vtctx->session);
+            vtctx->session = NULL;
+            status = vtenc_configure_encoder(avctx);
+            if (status == 0)
+                pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
+        }
+        if (!pix_buf_pool) {
+            av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
+            return AVERROR_EXTERNAL;
+        }
+        else
+            av_log(avctx, AV_LOG_WARNING, "VT session restarted because of a "
+                   "kVTInvalidSessionErr error.\n");
     }
 
     status = CVPixelBufferPoolCreatePixelBuffer(NULL,