[vlc-devel] [PATCH 17/30] avcodec: get the decoder device before creating the VA

[Steve Lhomme]

Only call decoder_HoldDecoderDevice() in lavc_UpdateVideoFormat(). The
decoder_UpdateVideoOutput() is done afterwards.

After the call to lavc_UpdateVideoFormat() the received decoder device holds an
extra reference that needs to be released after the VA is created.

decoder_NewPicture() cannot be used until decoder_UpdateVideoOutput() is called.
---
 modules/codec/avcodec/vaapi.c |  5 ++++-
 modules/codec/avcodec/video.c | 32 ++++++++++++++++++++++----------
 2 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/modules/codec/avcodec/vaapi.c b/modules/codec/avcodec/vaapi.c
index 7e8329bf888..d2bc5644725 100644
--- a/modules/codec/avcodec/vaapi.c
+++ b/modules/codec/avcodec/vaapi.c
@@ -166,10 +166,13 @@ static int Create(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
         vlc_vaapi_PicSysHoldInstance(p_sys, &va_dpy);
 
     VASurfaceID *render_targets;
-    unsigned num_render_targets =
+    unsigned num_render_targets;
+#if 0 // TODO create the render targets locally
+    num_render_targets =
         vlc_vaapi_PicSysGetRenderTargets(p_sys, &render_targets);
     if (num_render_targets == 0)
         goto error;
+#endif
 
     VAProfile i_profile;
     unsigned count;
diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 2232b7a291c..80d2de4eae4 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -341,7 +341,8 @@ static int lavc_GetVideoFormat(decoder_t *dec, video_format_t *restrict fmt,
 
 static int lavc_UpdateVideoFormat(decoder_t *dec, AVCodecContext *ctx,
                                   enum AVPixelFormat fmt,
-                                  enum AVPixelFormat swfmt)
+                                  enum AVPixelFormat swfmt,
+                                  vlc_decoder_device **dec_device)
 {
     video_format_t fmt_out;
     int val;
@@ -375,7 +376,7 @@ static int lavc_UpdateVideoFormat(decoder_t *dec, AVCodecContext *ctx,
         dec->fmt_out.video.mastering = dec->fmt_in.video.mastering;
     dec->fmt_out.video.lighting = dec->fmt_in.video.lighting;
 
-    return decoder_UpdateVideoFormat(dec);
+    return decoder_HoldDecoderDevice(dec, dec_device);
 }
 
 static bool chroma_compatible(vlc_fourcc_t a, vlc_fourcc_t b)
@@ -1247,7 +1248,8 @@ static int DecodeBlock( decoder_t *p_dec, block_t **pp_block )
              * then picture buffer can be allocated. */
             if (p_sys->p_va == NULL
              && lavc_UpdateVideoFormat(p_dec, p_context, p_context->pix_fmt,
-                                       p_context->pix_fmt) == 0)
+                                       p_context->pix_fmt, NULL) == 0
+             && decoder_UpdateVideoOutput(p_dec) == 0)
                 p_pic = decoder_NewPicture(p_dec);
 
             if( !p_pic )
@@ -1595,7 +1597,8 @@ static int lavc_GetFrame(struct AVCodecContext *ctx, AVFrame *frame, int flags)
         /* Most unaccelerated decoders do not call get_format(), so we need to
          * update the output video format here. The MT semaphore must be held
          * to protect p_dec->fmt_out. */
-        if (lavc_UpdateVideoFormat(dec, ctx, ctx->pix_fmt, ctx->pix_fmt))
+        if (lavc_UpdateVideoFormat(dec, ctx, ctx->pix_fmt, ctx->pix_fmt, NULL) ||
+            decoder_UpdateVideoOutput(dec))
         {
             post_mt(sys);
             return -1;
@@ -1728,24 +1731,33 @@ no_reuse:
             continue;
         }
         const AVPixFmtDescriptor *dsc = av_pix_fmt_desc_get(hwfmt);
+        vlc_decoder_device *init_device;
         msg_Dbg(p_dec, "trying format %s", dsc ? dsc->name : "unknown");
-        if (lavc_UpdateVideoFormat(p_dec, p_context, hwfmt, swfmt))
+        if (lavc_UpdateVideoFormat(p_dec, p_context, hwfmt, swfmt, &init_device))
             continue; /* Unsupported brand of hardware acceleration */
         post_mt(p_sys);
 
-        picture_t *test_pic = decoder_NewPicture(p_dec);
-        assert(!test_pic || test_pic->format.i_chroma == p_dec->fmt_out.video.i_chroma);
+        // TEMP: decoder_NewPicture cannot be used until decoder_UpdateVideoOutput is called
+        picture_t *test_pic = NULL;
         vlc_va_t *va = vlc_va_New(VLC_OBJECT(p_dec), p_context, hwfmt,
                                   &p_dec->fmt_in,
-                                  test_pic ? test_pic->p_sys : NULL);
-        if (test_pic)
-            picture_Release(test_pic);
+                                  NULL);
+        if (init_device)
+            vlc_decoder_device_Release(init_device);
         if (va == NULL)
         {
             wait_mt(p_sys);
             continue; /* Unsupported codec profile or such */
         }
 
+        if (decoder_UpdateVideoOutput(p_dec))
+        {
+            vlc_va_Delete(va);
+            p_context->hwaccel_context = NULL;
+            wait_mt(p_sys);
+            continue; /* Unsupported codec profile or such */
+        }
+
         p_sys->p_va = va;
         p_sys->pix_fmt = hwfmt;
         p_context->draw_horiz_band = NULL;
-- 
2.17.1