[vlc-devel] [PATCH] contrib/faad2: add patch to prevent crash on SCE followed by CPE

[Filip Roséen]

fixes: #18128

--

Resources describing the bug (it has been present but without fix for
a very long time):

 - https://trac.videolan.org/vlc/ticket/18128
 - https://forum.videolan.org/viewtopic.php?t=16725&start=40#p55357
 - https://trac.mplayerhq.hu/ticket/1323
 - https://bugs.launchpad.net/ubuntu/+source/banshee/+bug/269631
---
 .../src/faad2/faad2-fix-cpe-reconstruction.patch   | 37 ++++++++++++++++++++++
 contrib/src/faad2/rules.mak                        |  1 +
 2 files changed, 38 insertions(+)
 create mode 100644 contrib/src/faad2/faad2-fix-cpe-reconstruction.patch

diff --git a/contrib/src/faad2/faad2-fix-cpe-reconstruction.patch b/contrib/src/faad2/faad2-fix-cpe-reconstruction.patch
new file mode 100644
index 0000000000..9a345863d3
--- /dev/null
+++ b/contrib/src/faad2/faad2-fix-cpe-reconstruction.patch
@@ -0,0 +1,37 @@
+hDecoder->element_alloced denotes whether or not we have allocated memory for
+usage in terms of the specified channel element. Given that it previously only
+had two states (1 meaning allocated, and 0 meaning not allocated), it would not
+allocate enough memory for parsing a CPE it if is preceeded by a SCE (and
+therefor crash).
+
+These changes fixes the issue by making sure that we allocate additional memory
+if so is necessary, and the set of values for hDecoder->element_alloced[n] is
+now:
+
+ 0 = nothing allocated
+ 1 = allocated enough for SCE
+ 2 = allocated enough for CPE
+
+All branches that depend on hDecoder->element_alloced[n] prior to this patch
+only checks if the value is, or is not, zero. The added state, 2, is therefor
+correctly handled automatically.
+
+--- faad2/libfaad/specrec.c	2017-03-12 20:16:58.305887783 +0100
++++ faad2/libfaad/specrec.c	2017-03-25 03:48:19.875321643 +0100
+@@ -1110,13 +1110,13 @@
+ #ifdef PROFILE
+     int64_t count = faad_get_ts();
+ #endif
+-    if (hDecoder->element_alloced[hDecoder->fr_ch_ele] == 0)
++    if (hDecoder->element_alloced[hDecoder->fr_ch_ele] != 2)
+     {
+         retval = allocate_channel_pair(hDecoder, cpe->channel, (uint8_t)cpe->paired_channel);
+         if (retval > 0)
+             return retval;
+ 
+-        hDecoder->element_alloced[hDecoder->fr_ch_ele] = 1;
++        hDecoder->element_alloced[hDecoder->fr_ch_ele] = 2;
+     }
+ 
+     /* dequantisation and scaling */
+
diff --git a/contrib/src/faad2/rules.mak b/contrib/src/faad2/rules.mak
index f07f01d279..6fd73668cf 100644
--- a/contrib/src/faad2/rules.mak
+++ b/contrib/src/faad2/rules.mak
@@ -22,6 +22,7 @@ ifndef HAVE_FPU
 endif
 	$(APPLY) $(SRC)/faad2/faad2-disable-drc.patch
 	$(APPLY) $(SRC)/faad2/faad2-fix-71wPCEmapping.patch
+	$(APPLY) $(SRC)/faad2/faad2-fix-cpe-reconstruction.patch
 	cd $(UNPACK_DIR) && $(CC) -iquote . -E - </dev/null || sed -i 's/-iquote /-I/' libfaad/Makefile.am
 	$(MOVE)
 
-- 
2.12.1