[vlc-devel] [PATCH 1/2] avcodec: refactor PixelFormat reordering

[Thomas Guillem]

---
 modules/codec/avcodec/video.c | 50 +++++++++++++++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/modules/codec/avcodec/video.c b/modules/codec/avcodec/video.c
index 4dee8dd7fd..1837afe310 100644
--- a/modules/codec/avcodec/video.c
+++ b/modules/codec/avcodec/video.c
@@ -1488,6 +1488,40 @@ static int lavc_GetFrame(struct AVCodecContext *ctx, AVFrame *frame, int flags)
     return ret;
 }
 
+static void ReorderHwFmts(enum PixelFormat *pi_fmt, size_t count)
+{
+    static const enum PixelFormat fmt_order[] = {
+#if defined(_WIN32) && LIBAVUTIL_VERSION_CHECK(54, 13, 1, 24, 100)
+        /* Favor D3D11VA over DXVA2 */
+        AV_PIX_FMT_D3D11VA_VLD,
+        AV_PIX_FMT_DXVA2_VLD,
+#endif
+    };
+    static const size_t fmt_order_count = ARRAY_SIZE(fmt_order);
+
+    if (fmt_order_count <= 1)
+        return;
+
+    size_t order = 0;
+    for (size_t i = 0; i < fmt_order_count; ++i)
+    {
+        for (size_t j = 0; j < count; ++j)
+        {
+            if (fmt_order[i] == pi_fmt[j])
+            {
+                /* Remove the current fmt_order[i] from pi_fmt */
+                memmove(&pi_fmt[j], &pi_fmt[j + 1],
+                        (count - j) * sizeof(enum PixelFormat));
+                /* Create place holder for fmt_order[i] */
+                memmove(&pi_fmt[order + 1], &pi_fmt[order],
+                        (count - order) * sizeof(enum PixelFormat));
+                /* Move fmt_order[i] */
+                pi_fmt[order++] = fmt_order[i];
+            }
+        }
+    }
+}
+
 static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
                                           const enum PixelFormat *pi_fmt )
 {
@@ -1511,20 +1545,14 @@ static enum PixelFormat ffmpeg_GetFormat( AVCodecContext *p_context,
         if (hwaccel)
             can_hwaccel = true;
     }
-#if defined(_WIN32) && LIBAVUTIL_VERSION_CHECK(54, 13, 1, 24, 100)
+
     size_t count;
     for (count = 0; pi_fmt[count] != AV_PIX_FMT_NONE; count++);
     enum PixelFormat p_fmts[count + 1];
-    if (pi_fmt[0] == AV_PIX_FMT_DXVA2_VLD && pi_fmt[1] == AV_PIX_FMT_D3D11VA_VLD)
-    {
-        /* favor D3D11VA over DXVA2 as the order will decide which vout will be
-         * used */
-        memcpy(p_fmts, pi_fmt, sizeof(p_fmts));
-        p_fmts[0] = AV_PIX_FMT_D3D11VA_VLD;
-        p_fmts[1] = AV_PIX_FMT_DXVA2_VLD;
-        pi_fmt = p_fmts;
-    }
-#endif
+    memcpy(p_fmts, pi_fmt, sizeof(p_fmts));
+
+    ReorderHwFmts(p_fmts, count);
+    pi_fmt = p_fmts;
 
     /* If the format did not actually change (e.g. seeking), try to reuse the
      * existing output format, and if present, hardware acceleration back-end.
-- 
2.11.0