[vlc-devel] [PATCH 20/21] Push a sentinel buffer on the input buffer fifo when the output needs reconfiguring

[Martin Storsjö]

This avoids deadlocks on HTC Sensation, where the input buffers
aren't returned to the caller until the decoder has been
reconfigured.

Since all input buffers can be used up before the reconfigure event
is received, this earlier led to a deadlock where the decoder
waited for the caller to reconfigure it (as a response to the
reconfigure event), while the caller waited for an input buffer
to be freed to put the next input packet into.
---
 modules/codec/omxil/omxil.cpp |    8 ++++++++
 modules/codec/omxil/omxil.h   |    2 ++
 2 files changed, 10 insertions(+), 0 deletions(-)

diff --git a/modules/codec/omxil/omxil.cpp b/modules/codec/omxil/omxil.cpp
index 63a7a84..592f988 100644
--- a/modules/codec/omxil/omxil.cpp
+++ b/modules/codec/omxil/omxil.cpp
@@ -1290,6 +1290,10 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
 
     /* Send the input buffer to the component */
     OMX_FIFO_GET(&p_sys->in.fifo, p_header);
+
+    if (p_header && p_header->nFlags & OMX_BUFFERFLAG_EOS)
+        goto reconfig;
+
     if(p_header)
     {
         p_header->nFilledLen = p_block->i_buffer;
@@ -1343,6 +1347,7 @@ static picture_t *DecodeVideo( decoder_t *p_dec, block_t **pp_block )
         *pp_block = NULL; /* Avoid being fed the same packet again */
     }
 
+reconfig:
     /* Handle the PortSettingsChanged events */
     for(i = 0; i < p_sys->ports; i++)
     {
@@ -1654,6 +1659,9 @@ static OMX_ERRORTYPE OmxEventHandler( OMX_HANDLETYPE omx_handle,
         for(i = 0; i < p_sys->ports; i++)
             if(p_sys->p_ports[i].definition.eDir == OMX_DirOutput)
                 p_sys->p_ports[i].b_reconfigure = OMX_TRUE;
+        memset(&p_sys->sentinel_buffer, 0, sizeof(p_sys->sentinel_buffer));
+        p_sys->sentinel_buffer.nFlags = OMX_BUFFERFLAG_EOS;
+        OMX_FIFO_PUT(&p_sys->in.fifo, &p_sys->sentinel_buffer);
         break;
 
     default:
diff --git a/modules/codec/omxil/omxil.h b/modules/codec/omxil/omxil.h
index 6486a33..fa44b94 100644
--- a/modules/codec/omxil/omxil.h
+++ b/modules/codec/omxil/omxil.h
@@ -130,6 +130,8 @@ struct decoder_sys_t
     date_t end_date;
 
     int i_nal_size_length; /* Length of the NAL size field for H264 */
+
+    OMX_BUFFERHEADERTYPE sentinel_buffer;
 };
 
 #ifdef USE_IOMX
-- 
1.7.2.5