[vlc-devel] [PATCH 20/42] avcodec: get the decoder device before creating the VA

[Steve Lhomme]

Only call decoder_GetDecoderDevice() in lavc_UpdateVideoFormat(). The
decoder_UpdateVideoOutput() is done afterwards.

After the call to lavc_UpdateVideoFormat() the received decoder device holds an
extra reference that needs to be released after the VA is created.

decoder_NewPicture() cannot be used until decoder_UpdateVideoOutput() is called.
So this patch temporarily breaks va modules relying on the picture_sys_t data
that used to be passed. It will be via the decoder device we just got.
---
 modules/codec/avcodec/vaapi.c |  5 ++++-
 modules/codec/avcodec/video.c | 39 ++++++++++++++++++++++++++---------
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/modules/codec/avcodec/vaapi.c b/modules/codec/avcodec/vaapi.c
index be0a321b0e5..a374cedd507 100644
--- a/modules/codec/avcodec/vaapi.c
+++ b/modules/codec/avcodec/vaapi.c
@@ -168,10 +168,13 @@ static int Create(vlc_va_t *va, AVCodecContext *ctx, const AVPixFmtDescriptor *d
         vlc_vaapi_PicSysHoldInstance(p_sys, &va_dpy);
 
     VASurfaceID *render_targets;
-    unsigned num_render_targets =
+    unsigned num_render_targets;
+#if 0 // TODO create the render targets locally
+    num_render_targets =
         vlc_vaapi_PicSysGetRenderTargets(p_sys, &render_targets);
     if (num_render_targets == 0)
         goto error;
+#endif
 
     VAProfile i_profile;
     unsigned count;
diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 0b51cd940b7..9c2090e6024 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -333,7 +333,8 @@ static int lavc_GetVideoFormat(decoder_t *dec, video_format_t *restrict fmt,
 
 static int lavc_UpdateVideoFormat(decoder_t *dec, AVCodecContext *ctx,
                                   enum AVPixelFormat fmt,
-                                  enum AVPixelFormat swfmt)
+                                  enum AVPixelFormat swfmt,
+                                  vlc_decoder_device **pp_dec_device)
 {
     video_format_t fmt_out;
     int val;
@@ -367,7 +368,15 @@ static int lavc_UpdateVideoFormat(decoder_t *dec, AVCodecContext *ctx,
         dec->fmt_out.video.mastering = dec->fmt_in.video.mastering;
     dec->fmt_out.video.lighting = dec->fmt_in.video.lighting;
 
-    return decoder_UpdateVideoFormat(dec);
+    vlc_decoder_device *dec_dev = decoder_GetDecoderDevice(dec);
+    if (dec_dev)
+    {
+        if (pp_dec_device)
+            *pp_dec_device = dec_dev;
+        else
+            vlc_decoder_device_Release(dec_dev);
+    }
+    return 0;
 }
 
 static bool chroma_compatible(vlc_fourcc_t a, vlc_fourcc_t b)
@@ -1239,7 +1248,8 @@ static int DecodeBlock( decoder_t *p_dec, block_t **pp_block )
              * then picture buffer can be allocated. */
             if (p_sys->p_va == NULL
              && lavc_UpdateVideoFormat(p_dec, p_context, p_context->pix_fmt,
-                                       p_context->pix_fmt) == 0)
+                                       p_context->pix_fmt, NULL) == 0
+             && decoder_UpdateVideoOutput(p_dec) == 0)
                 p_pic = decoder_NewPicture(p_dec);
 
             if( !p_pic )
@@ -1592,7 +1602,8 @@ static int lavc_GetFrame(struct AVCodecContext *ctx, AVFrame *frame, int flags)
         /* Most unaccelerated decoders do not call get_format(), so we need to
          * update the output video format here. The MT semaphore must be held
          * to protect p_dec->fmt_out. */
-        if (lavc_UpdateVideoFormat(dec, ctx, ctx->pix_fmt, ctx->pix_fmt))
+        if (lavc_UpdateVideoFormat(dec, ctx, ctx->pix_fmt, ctx->pix_fmt, NULL) ||
+            decoder_UpdateVideoOutput(dec))
         {
             vlc_mutex_unlock(&sys->lock);
             return -1;
@@ -1731,22 +1742,30 @@ no_reuse:
             continue;
         }
         const AVPixFmtDescriptor *dsc = av_pix_fmt_desc_get(hwfmt);
+        vlc_decoder_device *init_device = NULL;
         msg_Dbg(p_dec, "trying format %s", dsc ? dsc->name : "unknown");
-        if (lavc_UpdateVideoFormat(p_dec, p_context, hwfmt, swfmt))
+        if (lavc_UpdateVideoFormat(p_dec, p_context, hwfmt, swfmt, &init_device) ||
+            init_device == NULL)
             continue; /* Unsupported brand of hardware acceleration */
         vlc_mutex_unlock(&p_sys->lock);
 
-        picture_t *test_pic = decoder_NewPicture(p_dec);
-        assert(!test_pic || test_pic->format.i_chroma == p_dec->fmt_out.video.i_chroma);
+        // TEMP: decoder_NewPicture cannot be used until decoder_UpdateVideoOutput is called
         vlc_va_t *va = vlc_va_New(VLC_OBJECT(p_dec), p_context, src_desc, hwfmt,
                                   &p_dec->fmt_in,
-                                  test_pic ? test_pic->p_sys : NULL);
-        if (test_pic)
-            picture_Release(test_pic);
+                                  NULL);
+        if (init_device)
+            vlc_decoder_device_Release(init_device);
         vlc_mutex_lock(&p_sys->lock);
         if (va == NULL)
             continue; /* Unsupported codec profile or such */
 
+        if (decoder_UpdateVideoOutput(p_dec))
+        {
+            vlc_va_Delete(va);
+            p_context->hwaccel_context = NULL;
+            continue; /* Unsupported codec profile or such */
+        }
+
         p_sys->p_va = va;
         p_sys->pix_fmt = hwfmt;
         p_context->draw_horiz_band = NULL;
-- 
2.17.1