[vlc-devel] [PATCH] [RFC] pixel shader to write I420 data in NV12 textures

[Steve Lhomme]

Direct3D11 doesn't support writing pixel formats in 3 planes.
So we write the U & V planes in the second texture, like for NV12 but use a pixel
shader to remap the values.

This is almost working, but some colours are slightly washed, yellow is absent :(

If anyone has more knowledge on pixel shaders, feel free to step in.
---
 modules/video_output/msw/direct3d11.c | 43 +++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/modules/video_output/msw/direct3d11.c b/modules/video_output/msw/direct3d11.c
index 952c309..a77d954 100644
--- a/modules/video_output/msw/direct3d11.c
+++ b/modules/video_output/msw/direct3d11.c
@@ -139,6 +139,46 @@ static const char* globVertexShaderDefault = "\
   }\
 "
 
+#define globPixelShaderBiplanarI420RGB "\
+  Texture2D shaderTextureY;\
+  Texture2D shaderTextureUV;\
+  SamplerState SampleType;\
+  \
+  struct PS_INPUT\
+  {\
+    float4 Position   : SV_POSITION;\
+    float2 Texture    : TEXCOORD0;\
+  };\
+  \
+  float4 PS( PS_INPUT In ) : SV_TARGET\
+  {\
+    float Y;\
+    float UCb;\
+    float VCr;\
+    float2 UCbPos;\
+    float2 VCrPos;\
+    float4 rgba;\
+    \
+    Y  = shaderTextureY.Sample(SampleType, In.Texture).x;\
+    \
+    UCbPos = In.Texture / 2;\
+    VCrPos = In.Texture / 2;\
+    VCrPos.x = VCrPos.x + 0.5;\
+    UCb = shaderTextureUV.Sample(SampleType, UCbPos).x;\
+    VCr = shaderTextureUV.Sample(SampleType, VCrPos).x;\
+    \
+    Y = 1.164383 * (Y - 0.0625);\
+    UCb = 0.5 - UCb;\
+    VCr = VCr - 0.5;\
+    \
+    rgba.x = saturate(Y + 1.370705 * VCr);\
+    rgba.y = saturate(Y - 0.698001 * VCr - 0.337633 * UCb);\
+    rgba.z = saturate(Y + 1.732446 * UCb);\
+    rgba.w = 1.0;\
+    return rgba;\
+  }\
+"
+
 #define globPixelShaderBiplanarYUV2RGB "\
   Texture2D shaderTextureY;\
   Texture2D shaderTextureUV;\
@@ -178,6 +218,9 @@ typedef struct
 } d3d_format_t;
 
 static const d3d_format_t d3d_formats[] = {
+#ifdef I420_SHADER
+    { "I420",     DXGI_FORMAT_NV12,           VLC_CODEC_I420,     DXGI_FORMAT_R8_UNORM,           DXGI_FORMAT_R8G8_UNORM, globPixelShaderBiplanarI420RGB },
+#endif
     { "NV12",     DXGI_FORMAT_NV12,           VLC_CODEC_NV12,     DXGI_FORMAT_R8_UNORM,           DXGI_FORMAT_R8G8_UNORM, globPixelShaderBiplanarYUV2RGB },
 #ifdef BROKEN_PIXEL
     { "YUY2",     DXGI_FORMAT_YUY2,           VLC_CODEC_I422,     DXGI_FORMAT_R8G8B8A8_UNORM,     0, globPixelShaderDefault },
-- 
2.3.0