[vlc-devel] [WIP/PATCH] avcodec: ensure correct buffer size

[Tristan Matthews]

Given this sample:
https://trac.videolan.org/vlc/attachment/ticket/6505/amono8000sps_10sec.wav
transcoding to ulaw (via avcodec) fails with repeated "filling error on encode"
messages.

This patch tries to address this by resizing the target buffers since
avcodec_fill_audio_frame will reject buffers that are too small.

Tested with:
./vlc -vvv -I "dummy" amono8000sps_10sec.wav  --sout '#transcode{acodec=ulaw,samplerate=8000,channels=1}:std{mux=wav,dst=out.wav}'

---
 modules/codec/avcodec/encoder.c | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/modules/codec/avcodec/encoder.c b/modules/codec/avcodec/encoder.c
index 8d1a790..e521ab5 100644
--- a/modules/codec/avcodec/encoder.c
+++ b/modules/codec/avcodec/encoder.c
@@ -1299,10 +1299,34 @@ static block_t *EncodeAudio( encoder_t *p_enc, block_t *p_aout_buf )
 
         }
 
+        /* ensure that buffer has acceptable size for avcodec */
+        const int needed_size = av_samples_get_buffer_size(NULL,
+                p_sys->p_context->channels, p_sys->frame->nb_samples,
+                p_sys->p_context->sample_fmt, align);
+
+        int buf_size = __MIN(p_sys->i_buffer_out, p_aout_buf->i_buffer);
+
+        if( unlikely( buf_size < needed_size ) )
+        {
+            if( p_sys->b_planar)
+            {
+                p_sys->p_buffer = realloc(p_sys->p_buffer, needed_size);
+                if( !p_sys->p_buffer )
+                    return p_chain;
+            }
+            else
+            {
+                p_aout_buf = block_Realloc(p_aout_buf, 0, needed_size);
+                if( !p_aout_buf )
+                    return p_chain;
+            }
+            buf_size = needed_size;
+        }
+
         if( avcodec_fill_audio_frame( p_sys->frame, p_sys->p_context->channels,
                                     p_sys->p_context->sample_fmt,
                                     p_sys->b_planar ? p_sys->p_buffer : p_aout_buf->p_buffer,
-                                    __MIN(p_sys->i_buffer_out, p_aout_buf->i_buffer),
+                                    buf_size,
                                     align) < 0 )
         {
                  msg_Err( p_enc, "filling error on encode" );
-- 
1.8.1.2