[vlc-commits] [Git][videolan/vlc][3.0.x] avcodec: video: fix swfmt detection when using a hw_device_ctx

[Steve Lhomme (@robUx4)]

Steve Lhomme pushed to branch 3.0.x at VideoLAN / VLC


Commits:
fae285f4 by Thomas Guillem at 2023-06-19T13:02:42+02:00
avcodec: video: fix swfmt detection when using a hw_device_ctx

(cherry picked from commit 906ffbaa631f54714ca55731b99f20d18100938e) (edited)
edited:
- 3.0 uses PixelFormat
- the code to handle b_hardware_only was already merged in 3.0

Signed-off-by: Steve Lhomme <robux4 at ycbcr.xyz>

- - - - -


1 changed file:

- modules/codec/avcodec/video.c


Changes:

=====================================
modules/codec/avcodec/video.c
=====================================
@@ -1725,7 +1725,8 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
     video_format_t fmt;
 
     /* Enumerate available formats */
-    enum PixelFormat swfmt = avcodec_default_get_format(p_context, pi_fmt);
+    enum PixelFormat defaultfmt = avcodec_default_get_format(p_context, pi_fmt);
+    enum PixelFormat swfmt = AV_PIX_FMT_NONE;
     bool can_hwaccel = false;
 
     for (size_t i = 0; pi_fmt[i] != AV_PIX_FMT_NONE; i++)
@@ -1738,13 +1739,22 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
         msg_Dbg( p_dec, "available %sware decoder output format %d (%s)",
                  hwaccel ? "hard" : "soft", pi_fmt[i], dsc->name );
         if (hwaccel)
+        {
+            /* The default fmt is a hw format, it can happen with some va
+             * implementations (when using a hw_device_ctx). */
+            if (defaultfmt == pi_fmt[i])
+                defaultfmt = AV_PIX_FMT_NONE;
+
             can_hwaccel = true;
+        }
+        else if (swfmt == AV_PIX_FMT_NONE && !p_sys->b_hardware_only)
+            swfmt = pi_fmt[i];
     }
 
     /* Use the default fmt in priority of any sw fmt if the default fmt is a hw
      * one */
-    if (p_sys->pix_fmt != AV_PIX_FMT_NONE && !p_sys->b_hardware_only)
-        swfmt = p_sys->pix_fmt;
+    if (defaultfmt != AV_PIX_FMT_NONE && !p_sys->b_hardware_only)
+        swfmt = defaultfmt;
 
     if (p_sys->pix_fmt == AV_PIX_FMT_NONE)
         goto no_reuse;



View it on GitLab: https://code.videolan.org/videolan/vlc/-/commit/fae285f41374cc477bd550ac48ac21e47e0dab10

-- 
View it on GitLab: https://code.videolan.org/videolan/vlc/-/commit/fae285f41374cc477bd550ac48ac21e47e0dab10
You're receiving this email because of your account on code.videolan.org.


VideoLAN code repository instance