[x265] [PATCH] encoder: ensure returned NAL count matches output array size

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1380054066 18000
#      Tue Sep 24 15:21:06 2013 -0500
# Node ID bdd26fd0325acf0f36409e994bdc262b11fa70f4
# Parent  bb5d3e9aadc340838e6af506d240fe0cb9d96c1d
encoder: ensure returned NAL count matches output array size

On CHECKED_MALLOC failure, m_packetData and/or m_nals will be NULL and thus the
returned count must be 0. Also, do not free the packet data from within this
utility function. It is cleaner to release that memory in the function which
declared the pointers on the stack.

diff -r bb5d3e9aadc3 -r bdd26fd0325a source/encoder/encoder.cpp
--- a/source/encoder/encoder.cpp	Tue Sep 24 14:59:59 2013 -0500
+++ b/source/encoder/encoder.cpp	Tue Sep 24 15:21:06 2013 -0500
@@ -443,20 +443,20 @@
 {
     uint32_t memsize = 0;
     uint32_t offset = 0;
+    int nalcount = 0;
 
-    int nalcount = 0;
-    for (; nalunits[nalcount] != NULL; nalcount++)
+    int num = 0;
+    for (; nalunits[num] != NULL; num++)
     {
-        const NALUnitEBSP& temp = *nalunits[nalcount];
+        const NALUnitEBSP& temp = *nalunits[num];
         memsize += temp.m_packetSize + 4;
     }
     
     X265_FREE(m_packetData);
     X265_FREE(m_nals);
     CHECKED_MALLOC(m_packetData, char, memsize);
-    CHECKED_MALLOC(m_nals, x265_nal_t, nalcount);
+    CHECKED_MALLOC(m_nals, x265_nal_t, num);
 
-    nalcount = 0;
     memsize = 0;
 
     /* Copy NAL output packets into x265_nal_t structures */
@@ -492,9 +492,6 @@
 
         m_nals[nalcount].i_type = nalu.m_nalUnitType;
         m_nals[nalcount].i_payload = size;
-        free(nalu.m_nalUnitData);
-        X265_FREE(nalunits[nalcount]);
-        nalunits[nalcount] = NULL;
     }
 
     /* Setup payload pointers, now that we're done adding content to m_packetData */