[vlc-commits] direct3d11: fix the luminance scaling at each step

[Steve Lhomme]

vlc/vlc-3.0 | branch: master | Steve Lhomme <robux4 at ycbcr.xyz> | Wed May 29 11:56:36 2019 +0200| [2874cb181a135317042ef50332bb8bacd8de2885] | committer: Steve Lhomme

direct3d11: fix the luminance scaling at each step

The conversion to Linear should end up with the max luminance, not
normalized to 0..1. The conversion from linear also assumes non normalized
pixel values.

The luminance scale also needs to be applied when not tone mapping (SDR to PQ).

(cherry picked from commit 594f02e8f6f48ce3df1e85a21baae83be1e43c50)

> http://git.videolan.org/gitweb.cgi/vlc/vlc-3.0.git/?a=commit;h=2874cb181a135317042ef50332bb8bacd8de2885
---

 modules/video_output/win32/d3d11_quad.c    |  2 +-
 modules/video_output/win32/d3d11_shaders.c | 25 ++++++++++---------------
 modules/video_output/win32/direct3d11.c    |  2 +-
 3 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/modules/video_output/win32/d3d11_quad.c b/modules/video_output/win32/d3d11_quad.c
index b113ed6e96..ca9d31bb59 100644
--- a/modules/video_output/win32/d3d11_quad.c
+++ b/modules/video_output/win32/d3d11_quad.c
@@ -826,7 +826,7 @@ int D3D11_SetupQuad(vlc_object_t *o, d3d11_device_t *d3d_dev, const video_format
     HRESULT hr;
     const bool RGB_shader = IsRGBShader(quad->formatInfo);
 
-    quad->shaderConstants.LuminanceScale = GetFormatLuminance(o, fmt) / (float)displayFormat->luminance_peak;
+    quad->shaderConstants.LuminanceScale = (float)displayFormat->luminance_peak / GetFormatLuminance(o, fmt);
 
     /* pixel shader constant buffer */
     quad->shaderConstants.Opacity = 1.0;
diff --git a/modules/video_output/win32/d3d11_shaders.c b/modules/video_output/win32/d3d11_shaders.c
index 81f605d1af..e79d711714 100644
--- a/modules/video_output/win32/d3d11_shaders.c
+++ b/modules/video_output/win32/d3d11_shaders.c
@@ -214,10 +214,9 @@ HRESULT D3D11_CompilePixelShader(vlc_object_t *o, d3d11_handle_t *hd3d, bool leg
     const char *psz_src_transform     = DEFAULT_NOOP;
     const char *psz_display_transform = DEFAULT_NOOP;
     const char *psz_primaries_transform = DEFAULT_NOOP;
-    const char *psz_tone_mapping      = DEFAULT_NOOP;
+    const char *psz_tone_mapping      = "return rgb * LuminanceScale";
     const char *psz_adjust_range      = DEFAULT_NOOP;
     char *psz_range = NULL;
-    char *psz_transform = NULL;
 
     switch (format->formatTexture)
     {
@@ -283,20 +282,17 @@ HRESULT D3D11_CompilePixelShader(vlc_object_t *o, d3d11_handle_t *hd3d, bool leg
                        "rgb = pow(max(rgb, 0), 1.0/ST2084_m2);\n"
                        "rgb = max(rgb - ST2084_c1, 0.0) / (ST2084_c2 - ST2084_c3 * rgb);\n"
                        "rgb = pow(rgb, 1.0/ST2084_m1);\n"
-                       "return rgb";
+                       "return rgb * 10000";
                 src_transfer = TRANSFER_FUNC_LINEAR;
                 break;
             case TRANSFER_FUNC_HLG:
-                asprintf(&psz_transform, "const float alpha_gain = 10000.0;\n"
-                                          "rgb.r = inverse_HLG(rgb.r);\n"
-                                          "rgb.g = inverse_HLG(rgb.g);\n"
-                                          "rgb.b = inverse_HLG(rgb.b);\n"
-                                          "float3 ootf_2020 = float3(0.2627, 0.6780, 0.0593);\n"
-                                          "float ootf_ys = alpha_gain * dot(ootf_2020, rgb);\n"
-                                          "rgb *= pow(ootf_ys, 0.200);\n"
-                                          "return rgb / %lld;",
-                         display->colorspace->transfer == TRANSFER_FUNC_SMPTE_ST2084 ? 1000 : 250);
-                psz_src_transform = psz_transform;
+                psz_src_transform = "const float alpha_gain = 2000; /* depends on the display output */\n"
+                                    "rgb.r = inverse_HLG(rgb.r);\n"
+                                    "rgb.g = inverse_HLG(rgb.g);\n"
+                                    "rgb.b = inverse_HLG(rgb.b);\n"
+                                    "float3 ootf_2020 = float3(0.2627, 0.6780, 0.0593);\n"
+                                    "float ootf_ys = alpha_gain * dot(ootf_2020, rgb);\n"
+                                    "return rgb * pow(ootf_ys, 0.200)";
                 src_transfer = TRANSFER_FUNC_LINEAR;
                 break;
             case TRANSFER_FUNC_BT709:
@@ -345,7 +341,7 @@ HRESULT D3D11_CompilePixelShader(vlc_object_t *o, d3d11_handle_t *hd3d, bool leg
                     /* Linear to ST2084 */
                     psz_display_transform =
                            ST2084_PQ_CONSTANTS
-                           "rgb = pow(rgb, ST2084_m1);\n"
+                           "rgb = pow(rgb / 10000, ST2084_m1);\n"
                            "rgb = (ST2084_c1 + ST2084_c2 * rgb) / (1 + ST2084_c3 * rgb);\n"
                            "rgb = pow(rgb, ST2084_m2);\n"
                            "return rgb";
@@ -476,7 +472,6 @@ HRESULT D3D11_CompilePixelShader(vlc_object_t *o, d3d11_handle_t *hd3d, bool leg
     }
 #endif
     free(psz_range);
-    free(psz_transform);
 
     ID3DBlob *pPSBlob = D3D11_CompileShader(o, hd3d, d3d_dev, shader, true);
     free(shader);
diff --git a/modules/video_output/win32/direct3d11.c b/modules/video_output/win32/direct3d11.c
index b2e484180d..99c0cf6dbf 100644
--- a/modules/video_output/win32/direct3d11.c
+++ b/modules/video_output/win32/direct3d11.c
@@ -946,7 +946,7 @@ static void Prepare(vout_display_t *vd, picture_t *picture, subpicture_t *subpic
 
     if (picture->format.mastering.max_luminance)
     {
-        D3D11_UpdateQuadLuminanceScale(vd, &sys->d3d_dev, &sys->picQuad, GetFormatLuminance(VLC_OBJECT(vd), &picture->format) / (float)sys->display.luminance_peak);
+        D3D11_UpdateQuadLuminanceScale(vd, &sys->d3d_dev, &sys->picQuad, (float)sys->display.luminance_peak / GetFormatLuminance(VLC_OBJECT(vd), &picture->format));
 
         if (sys->dxgiswapChain4)
         {