[vlc-commits] contrib/libav: add patch to fix memory leaks during VDA hardware decoding

[David Fuhrmann]

vlc/vlc-2.1 | branch: master | David Fuhrmann <david.fuhrmann at googlemail.com> | Wed Sep 11 20:44:50 2013 +0200| [469358dd2c735b67fdeacc81e1f35ec9643f132a] | committer: David Fuhrmann

contrib/libav: add patch to fix memory leaks during VDA hardware decoding

> http://git.videolan.org/gitweb.cgi/vlc/vlc-2.1.git/?a=commit;h=469358dd2c735b67fdeacc81e1f35ec9643f132a
---

 contrib/src/ffmpeg/fix-vda-memleak.patch |   73 ++++++++++++++++++++++++++++++
 contrib/src/ffmpeg/rules.mak             |    4 ++
 2 files changed, 77 insertions(+)

diff --git a/contrib/src/ffmpeg/fix-vda-memleak.patch b/contrib/src/ffmpeg/fix-vda-memleak.patch
new file mode 100644
index 0000000..c181a5e
--- /dev/null
+++ b/contrib/src/ffmpeg/fix-vda-memleak.patch
@@ -0,0 +1,73 @@
+From e952bd35716bd35c7e825cd8a18fc73aedc3edcc Mon Sep 17 00:00:00 2001
+From: Sebastien Zwickert <dilaroga at gmail.com>
+Date: Wed, 24 Jul 2013 11:36:01 +0200
+Subject: [PATCH] vda_h264: Use av_buffer to manage buffers
+
+Fix a buffer leak when seeking occurs.
+
+Signed-off-by: Luca Barbato <lu_zero at gentoo.org>
+---
+ libavcodec/vda_h264.c | 25 ++++++++++++++++++++++++-
+ libavcodec/version.h  |  2 +-
+ 2 files changed, 25 insertions(+), 2 deletions(-)
+
+diff --git a/libavcodec/vda_h264.c b/libavcodec/vda_h264.c
+index 6c1845a..7328d58 100644
+--- a/libavcodec/vda_h264.c
++++ b/libavcodec/vda_h264.c
+@@ -107,6 +107,12 @@ static int vda_h264_decode_slice(AVCodecContext *avctx,
+     return 0;
+ }
+ 
++static void vda_h264_release_buffer(void *opaque, uint8_t *data)
++{
++    CVPixelBufferRef cv_buffer = opaque;
++    CVPixelBufferRelease(cv_buffer);
++}
++
+ static int vda_h264_end_frame(AVCodecContext *avctx)
+ {
+     H264Context *h                      = avctx->priv_data;
+@@ -120,8 +126,25 @@ static int vda_h264_end_frame(AVCodecContext *avctx)
+     status = vda_sync_decode(vda_ctx);
+     frame->data[3] = (void*)vda_ctx->cv_buffer;
+ 
+-    if (status)
++    if (status) {
+         av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
++        return status;
++    }
++
++    /* VDA workaround to release properly each core video buffer:
++     * we need to create an extra av_buffer with a custom freeing callback
++     * to avoid potential memory leaks. */
++    h->cur_pic_ptr->hwaccel_priv_buf = av_buffer_create(frame->data[0],
++                                                        0,
++                                                        vda_h264_release_buffer,
++                                                        vda_ctx->cv_buffer,
++                                                        0);
++    if (!h->cur_pic_ptr->hwaccel_priv_buf) {
++        CVPixelBufferRelease(vda_ctx->cv_buffer);
++        return AVERROR(ENOMEM);
++    }
++
++    h->cur_pic_ptr->hwaccel_picture_private = h->cur_pic_ptr->hwaccel_priv_buf->data;
+ 
+     return status;
+ }
+diff --git a/libavcodec/version.h b/libavcodec/version.h
+index 980c5c5..9775a65 100644
+--- a/libavcodec/version.h
++++ b/libavcodec/version.h
+@@ -27,7 +27,7 @@
+  */
+ 
+ #define LIBAVCODEC_VERSION_MAJOR 55
+-#define LIBAVCODEC_VERSION_MINOR 17
++#define LIBAVCODEC_VERSION_MINOR 19
+ #define LIBAVCODEC_VERSION_MICRO  0
+ 
+ #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
+-- 
+1.7.12.4 (Apple Git-37)
+
diff --git a/contrib/src/ffmpeg/rules.mak b/contrib/src/ffmpeg/rules.mak
index 30f5dbb..848ffaa 100644
--- a/contrib/src/ffmpeg/rules.mak
+++ b/contrib/src/ffmpeg/rules.mak
@@ -136,6 +136,10 @@ ffmpeg: ffmpeg-$(HASH).tar.gz .sum-ffmpeg
 	rm -Rf $@ $@-$(HASH)
 	mkdir -p $@-$(HASH)
 	$(ZCAT) "$<" | (cd $@-$(HASH) && tar xv --strip-components=1)
+
+	# this patch is only needed for libav version b1f9cdc37ff5d5b391d2cd9af737ab4e5a0fc1c0
+	$(APPLY) $(SRC)/ffmpeg/fix-vda-memleak.patch
+
 	$(MOVE)
 
 .ffmpeg: ffmpeg