[vlc-commits] direct3d11: fix the luminance scaling at each step

[Steve Lhomme]

vlc | branch: master | Steve Lhomme <robux4 at ycbcr.xyz> | Wed May 29 11:56:36 2019 +0200| [2c7ca9f4edfaf99f6581b9ec86546b98ef03d9a3] | committer: Steve Lhomme

direct3d11: fix the luminance scaling at each step

The conversion to Linear should end up with the max luminance, not
normalized to 0..1. The conversion from linear also assumes non normalized
pixel values.

The luminance scale also needs to be applied when not tone mapping (SDR to PQ).

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=2c7ca9f4edfaf99f6581b9ec86546b98ef03d9a3
---

 modules/video_output/win32/d3d11_quad.c    |  2 +-
 modules/video_output/win32/d3d11_shaders.c | 26 +++++++++++---------------
 modules/video_output/win32/direct3d11.c    |  2 +-
 3 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/modules/video_output/win32/d3d11_quad.c b/modules/video_output/win32/d3d11_quad.c
index 9ee7661027..23406b1aaf 100644
--- a/modules/video_output/win32/d3d11_quad.c
+++ b/modules/video_output/win32/d3d11_quad.c
@@ -908,7 +908,7 @@ int D3D11_SetupQuad(vlc_object_t *o, d3d11_device_t *d3d_dev, const video_format
 {
     const bool RGB_shader = IsRGBShader(quad->textureFormat);
 
-    quad->shaderConstants.LuminanceScale = GetFormatLuminance(o, fmt) / (float)displayFormat->luminance_peak;
+    quad->shaderConstants.LuminanceScale = (float)displayFormat->luminance_peak / GetFormatLuminance(o, fmt);
 
     /* pixel shader constant buffer */
     quad->shaderConstants.Opacity = 1.0;
diff --git a/modules/video_output/win32/d3d11_shaders.c b/modules/video_output/win32/d3d11_shaders.c
index e516112c60..6eb8467e19 100644
--- a/modules/video_output/win32/d3d11_shaders.c
+++ b/modules/video_output/win32/d3d11_shaders.c
@@ -262,11 +262,10 @@ HRESULT D3D11_CompilePixelShader(vlc_object_t *o, d3d11_handle_t *hd3d, bool leg
     const char *psz_src_transform     = DEFAULT_NOOP;
     const char *psz_display_transform = DEFAULT_NOOP;
     const char *psz_primaries_transform = DEFAULT_NOOP;
-    const char *psz_tone_mapping      = DEFAULT_NOOP;
+    const char *psz_tone_mapping      = "return rgb * LuminanceScale";
     const char *psz_adjust_range      = DEFAULT_NOOP;
     const char *psz_move_planes[2]    = {DEFAULT_NOOP, DEFAULT_NOOP};
     char *psz_range = NULL;
-    char *psz_transform = NULL;
 
     D3D11_SAMPLER_DESC sampDesc;
     memset(&sampDesc, 0, sizeof(sampDesc));
@@ -436,20 +435,18 @@ HRESULT D3D11_CompilePixelShader(vlc_object_t *o, d3d11_handle_t *hd3d, bool leg
                        "rgb = pow(max(rgb, 0), 1.0/ST2084_m2);\n"
                        "rgb = max(rgb - ST2084_c1, 0.0) / (ST2084_c2 - ST2084_c3 * rgb);\n"
                        "rgb = pow(rgb, 1.0/ST2084_m1);\n"
-                       "return rgb";
+                       "return rgb * 10000";
                 src_transfer = TRANSFER_FUNC_LINEAR;
                 break;
             case TRANSFER_FUNC_HLG:
-                asprintf(&psz_transform, "const float alpha_gain = 10000.0;\n"
-                                          "rgb.r = inverse_HLG(rgb.r);\n"
-                                          "rgb.g = inverse_HLG(rgb.g);\n"
-                                          "rgb.b = inverse_HLG(rgb.b);\n"
-                                          "float3 ootf_2020 = float3(0.2627, 0.6780, 0.0593);\n"
-                                          "float ootf_ys = alpha_gain * dot(ootf_2020, rgb);\n"
-                                          "rgb *= pow(ootf_ys, 0.200);\n"
-                                          "return rgb / %lld;",
-                         display->transfer == TRANSFER_FUNC_SMPTE_ST2084 ? 1000 : 250);
-                psz_src_transform = psz_transform;
+                psz_src_transform = "const float alpha_gain = 2000; /* depends on the display output */\n"
+                                    "/* TODO: in one call */\n"
+                                    "rgb.r = inverse_HLG(rgb.r);\n"
+                                    "rgb.g = inverse_HLG(rgb.g);\n"
+                                    "rgb.b = inverse_HLG(rgb.b);\n"
+                                    "float3 ootf_2020 = float3(0.2627, 0.6780, 0.0593);\n"
+                                    "float ootf_ys = alpha_gain * dot(ootf_2020, rgb);\n"
+                                    "return rgb * pow(ootf_ys, 0.200)";
                 src_transfer = TRANSFER_FUNC_LINEAR;
                 break;
             case TRANSFER_FUNC_BT709:
@@ -498,7 +495,7 @@ HRESULT D3D11_CompilePixelShader(vlc_object_t *o, d3d11_handle_t *hd3d, bool leg
                     /* Linear to ST2084 */
                     psz_display_transform =
                            ST2084_PQ_CONSTANTS
-                           "rgb = pow(rgb, ST2084_m1);\n"
+                           "rgb = pow(rgb / 10000, ST2084_m1);\n"
                            "rgb = (ST2084_c1 + ST2084_c2 * rgb) / (1 + ST2084_c3 * rgb);\n"
                            "rgb = pow(rgb, ST2084_m2);\n"
                            "return rgb";
@@ -617,7 +614,6 @@ HRESULT D3D11_CompilePixelShader(vlc_object_t *o, d3d11_handle_t *hd3d, bool leg
                                  psz_display_transform, psz_tone_mapping,
                                  psz_adjust_range, psz_move_planes[1], &quad->d3dpixelShader[1]);
     free(psz_range);
-    free(psz_transform);
 
     return hr;
 }
diff --git a/modules/video_output/win32/direct3d11.c b/modules/video_output/win32/direct3d11.c
index 93282a9d92..72775bcc79 100644
--- a/modules/video_output/win32/direct3d11.c
+++ b/modules/video_output/win32/direct3d11.c
@@ -1075,7 +1075,7 @@ static void PreparePicture(vout_display_t *vd, picture_t *picture, subpicture_t
 
     if (picture->format.mastering.max_luminance)
     {
-        D3D11_UpdateQuadLuminanceScale(vd, &sys->d3d_dev, &sys->picQuad, GetFormatLuminance(VLC_OBJECT(vd), &picture->format) / (float)sys->display.luminance_peak);
+        D3D11_UpdateQuadLuminanceScale(vd, &sys->d3d_dev, &sys->picQuad, (float)sys->display.luminance_peak / GetFormatLuminance(VLC_OBJECT(vd), &picture->format));
     }
 
     /* Render the quad */