[vlc-devel] [PATCH] [WIP] RFC: add HDR/BT.2020 support (using libplacebo)
Niklas Haas
vlc at haasn.xyz
Thu Oct 26 00:59:56 CEST 2017
Attached is my preliminary patch for how to use libplacebo
(https://github.com/haasn/libplacebo) to convert between color spaces in
VLC, including HDR->SDR tone mapping. libplacebo is an ongoing project
to develop the GLSL rendering system from the mpv project into an
independent, reusable library.
Remaining issues that need to be resolved:
- It needs to be added to the build system. Due to the early nature of
libplacebo and the unstable API that results from it, it should be
configured to use a pinned commit and build the library statically.
- libplacebo requires LC_NUMERIC="C" (so `snprintf` works). I'm not sure
what the best way to accomplish this in VLC is; but one way is to use
`uselocale` from opengl/vout_helper.c to explicitly set the locale for
that thread. Alternatively, I could try making libplacebo set the
locale back and forth in every function so as to not depend on it, but
I'd rather not if I can avoid it.
- We should use libplacebo in more ways, for example to replace the
YUV->RGB conversion code (which would also allow VLC to support the
proper BT.2020 transformation including BT.2020-CL; and also allow VLC
to get rid of the special cased code for XYZ12). But I think this
would probably be better off as a second/separate commit.
Thoughts? Milestone?
-------------- next part --------------
>From 85d40590b279e0014ed21388e90c77262d41822b Mon Sep 17 00:00:00 2001
From: Niklas Haas <git at haasn.xyz>
Date: Wed, 11 Oct 2017 21:17:23 +0200
Subject: [PATCH] (WIP) vout_opengl: add HDR support using libplacebo
TODO:
- add libplacebo to build system
- make libplacebo release and bump ABI version?
- replace color matrix code by libplacebo's colorspace.c?
---
modules/video_output/opengl/converter.h | 16 ++++
modules/video_output/opengl/fragment_shaders.c | 114 +++++++++++++++++++++++++
modules/video_output/opengl/vout_helper.c | 28 ++++++
3 files changed, 158 insertions(+)
diff --git a/modules/video_output/opengl/converter.h b/modules/video_output/opengl/converter.h
index 3acbf6911d..487dfbb04d 100644
--- a/modules/video_output/opengl/converter.h
+++ b/modules/video_output/opengl/converter.h
@@ -21,6 +21,8 @@
#ifndef VLC_OPENGL_CONVERTER_H
#define VLC_OPENGL_CONVERTER_H
+#include <libplacebo/shaders.h>
+
#include "vout_helper.h"
#include <vlc_plugin.h>
@@ -64,9 +66,13 @@ typedef void (APIENTRY *PFNGLBUFFERSTORAGEPROC) (GLenum target, GLsizeiptr size,
# define PFNGLVERTEXATTRIBPOINTERPROC typeof(glVertexAttribPointer)*
# define PFNGLENABLEVERTEXATTRIBARRAYPROC typeof(glEnableVertexAttribArray)*
# define PFNGLUNIFORMMATRIX4FVPROC typeof(glUniformMatrix4fv)*
+# define PFNGLUNIFORMMATRIX3FVPROC typeof(glUniformMatrix3fv)*
+# define PFNGLUNIFORMMATRIX2FVPROC typeof(glUniformMatrix2fv)*
# define PFNGLUNIFORM4FVPROC typeof(glUniform4fv)*
# define PFNGLUNIFORM4FPROC typeof(glUniform4f)*
+# define PFNGLUNIFORM3FPROC typeof(glUniform3f)*
# define PFNGLUNIFORM2FPROC typeof(glUniform2f)*
+# define PFNGLUNIFORM1FPROC typeof(glUniform1f)*
# define PFNGLUNIFORM1IPROC typeof(glUniform1i)*
# define PFNGLCREATESHADERPROC typeof(glCreateShader)*
# define PFNGLSHADERSOURCEPROC typeof(glShaderSource)*
@@ -144,9 +150,13 @@ typedef struct {
PFNGLVERTEXATTRIBPOINTERPROC VertexAttribPointer;
PFNGLENABLEVERTEXATTRIBARRAYPROC EnableVertexAttribArray;
PFNGLUNIFORMMATRIX4FVPROC UniformMatrix4fv;
+ PFNGLUNIFORMMATRIX3FVPROC UniformMatrix3fv;
+ PFNGLUNIFORMMATRIX2FVPROC UniformMatrix2fv;
PFNGLUNIFORM4FVPROC Uniform4fv;
PFNGLUNIFORM4FPROC Uniform4f;
+ PFNGLUNIFORM3FPROC Uniform3f;
PFNGLUNIFORM2FPROC Uniform2f;
+ PFNGLUNIFORM1FPROC Uniform1f;
PFNGLUNIFORM1IPROC Uniform1i;
/* Program commands */
@@ -190,6 +200,9 @@ struct opengl_tex_converter_t
/* Pointer to object gl, set by the caller */
vlc_gl_t *gl;
+ /* libplacebo context, created by the caller (optional) */
+ struct pl_context *pl_ctx;
+
/* Function pointers to OpenGL functions, set by the caller */
const opengl_vtable_t *vt;
@@ -244,9 +257,12 @@ struct opengl_tex_converter_t
GLint TexSize[PICTURE_PLANE_MAX]; /* for GL_TEXTURE_RECTANGLE */
GLint Coefficients;
GLint FillColor;
+ GLint *pl_vars; /* for pl_sh_res */
} uloc;
bool yuv_color;
GLfloat yuv_coefficients[16];
+ struct pl_shader *pl_sh;
+ const struct pl_shader_res *pl_sh_res;
/* Private context */
void *priv;
diff --git a/modules/video_output/opengl/fragment_shaders.c b/modules/video_output/opengl/fragment_shaders.c
index 5de7b95359..375732cecb 100644
--- a/modules/video_output/opengl/fragment_shaders.c
+++ b/modules/video_output/opengl/fragment_shaders.c
@@ -24,6 +24,8 @@
#include <assert.h>
#include <stdlib.h>
+#include <libplacebo/shaders.h>
+#include <libplacebo/shaders/colorspace.h>
#include <vlc_common.h>
#include <vlc_memstream.h>
@@ -314,6 +316,12 @@ tc_base_fetch_locations(opengl_tex_converter_t *tc, GLuint program)
tc->uloc.FillColor = tc->vt->GetUniformLocation(program, "FillColor");
if (tc->uloc.FillColor == -1)
return VLC_EGENERIC;
+
+ const struct pl_shader_res *res = tc->pl_sh_res;
+ for (int i = 0; res && i < res->num_variables; i++) {
+ struct pl_shader_var sv = res->variables[i];
+ tc->uloc.pl_vars[i] = tc->vt->GetUniformLocation(program, sv.var.name);
+ }
return VLC_SUCCESS;
}
@@ -338,6 +346,38 @@ tc_base_prepare_shader(const opengl_tex_converter_t *tc,
tc->vt->Uniform2f(tc->uloc.TexSize[i], tex_width[i],
tex_height[i]);
}
+
+ const struct pl_shader_res *res = tc->pl_sh_res;
+ for (int i = 0; res && i < res->num_variables; i++) {
+ GLint loc = tc->uloc.pl_vars[i];
+ if (loc == -1) // uniform optimized out
+ continue;
+
+ struct pl_shader_var sv = res->variables[i];
+ struct ra_var var = sv.var;
+
+ // libplacebo doesn't need anything else anyway
+ if (var.type != RA_VAR_FLOAT)
+ continue;
+ if (var.dim_m > 1 && var.dim_m != var.dim_v)
+ continue;
+
+ const float *f = sv.data;
+ switch (var.dim_m) {
+ case 4: tc->vt->UniformMatrix4fv(loc, 1, GL_FALSE, f); break;
+ case 3: tc->vt->UniformMatrix3fv(loc, 1, GL_FALSE, f); break;
+ case 2: tc->vt->UniformMatrix2fv(loc, 1, GL_FALSE, f); break;
+
+ case 1:
+ switch (var.dim_v) {
+ case 1: tc->vt->Uniform1f(loc, f[0]); break;
+ case 2: tc->vt->Uniform2f(loc, f[0], f[1]); break;
+ case 3: tc->vt->Uniform3f(loc, f[0], f[1], f[2]); break;
+ case 4: tc->vt->Uniform4f(loc, f[0], f[1], f[2], f[3]); break;
+ }
+ break;
+ }
+ }
}
static int
@@ -411,6 +451,51 @@ xyz12_shader_init(opengl_tex_converter_t *tc)
return fragment_shader;
}
+static struct pl_color_space pl_color_space_from_video_format(video_format_t fmt)
+{
+ static enum pl_color_primaries primaries[COLOR_PRIMARIES_MAX+1] = {
+ [COLOR_PRIMARIES_UNDEF] = PL_COLOR_PRIM_UNKNOWN,
+ [COLOR_PRIMARIES_BT601_525] = PL_COLOR_PRIM_BT_601_525,
+ [COLOR_PRIMARIES_BT601_625] = PL_COLOR_PRIM_BT_601_625,
+ [COLOR_PRIMARIES_BT709] = PL_COLOR_PRIM_BT_709,
+ [COLOR_PRIMARIES_BT2020] = PL_COLOR_PRIM_BT_2020,
+ [COLOR_PRIMARIES_DCI_P3] = PL_COLOR_PRIM_DCI_P3,
+ [COLOR_PRIMARIES_BT470_M] = PL_COLOR_PRIM_BT_470M,
+ };
+
+ static enum pl_color_transfer transfers[TRANSFER_FUNC_MAX+1] = {
+ [TRANSFER_FUNC_UNDEF] = PL_COLOR_TRC_UNKNOWN,
+ [TRANSFER_FUNC_LINEAR] = PL_COLOR_TRC_LINEAR,
+ [TRANSFER_FUNC_SRGB] = PL_COLOR_TRC_SRGB,
+ [TRANSFER_FUNC_SMPTE_ST2084] = PL_COLOR_TRC_PQ,
+ [TRANSFER_FUNC_HLG] = PL_COLOR_TRC_HLG,
+ // these are all designed to be displayed on BT.1886 displays, so this
+ // is the correct way to handle them in libplacebo
+ [TRANSFER_FUNC_BT470_BG] = PL_COLOR_TRC_BT_1886,
+ [TRANSFER_FUNC_BT470_M] = PL_COLOR_TRC_BT_1886,
+ [TRANSFER_FUNC_BT709] = PL_COLOR_TRC_BT_1886,
+ [TRANSFER_FUNC_SMPTE_240] = PL_COLOR_TRC_BT_1886, // XXX: double-check?
+ };
+
+ // Derive the signal peak from the mastering metadata / content light
+ // metadata if available.
+ float sig_peak = 0.0;
+ if (fmt.mastering.max_luminance) {
+ sig_peak = fmt.mastering.max_luminance / PL_COLOR_REF_WHITE;
+ } else if (fmt.lighting.MaxCLL) {
+ sig_peak = fmt.lighting.MaxCLL / PL_COLOR_REF_WHITE;
+ }
+
+ return (struct pl_color_space) {
+ .primaries = primaries[fmt.primaries],
+ .transfer = transfers[fmt.transfer],
+ .light = fmt.transfer == TRANSFER_FUNC_HLG
+ ? PL_COLOR_LIGHT_SCENE_HLG
+ : PL_COLOR_LIGHT_DISPLAY,
+ .sig_peak = sig_peak,
+ };
+}
+
GLuint
opengl_fragment_shader_init_impl(opengl_tex_converter_t *tc, GLenum tex_target,
vlc_fourcc_t chroma, video_color_space_t yuv_space)
@@ -462,6 +547,28 @@ opengl_fragment_shader_init_impl(opengl_tex_converter_t *tc, GLenum tex_target,
ADDF("uniform %s Texture%u;"
"varying vec2 TexCoord%u;", sampler, i, i);
+ if (tc->pl_sh) {
+ struct pl_shader *sh = tc->pl_sh;
+ pl_shader_color_map(sh, &pl_color_map_default_params,
+ pl_color_space_from_video_format(tc->fmt),
+ pl_color_space_unknown, false);
+
+ const struct pl_shader_res *res = tc->pl_sh_res = pl_shader_finalize(sh);
+
+ FREENULL(tc->uloc.pl_vars);
+ tc->uloc.pl_vars = calloc(res->num_variables, sizeof(GLint));
+ for (int i = 0; i < res->num_variables; i++) {
+ struct pl_shader_var sv = res->variables[i];
+ ADDF("uniform %s %s;", ra_var_glsl_type_name(sv.var), sv.var.name);
+ }
+
+ // We can't handle these yet, but nothing we use requires them, either
+ assert(res->num_vertex_attribs == 0);
+ assert(res->num_descriptors == 0);
+
+ ADD(res->glsl);
+ }
+
if (tex_target == GL_TEXTURE_RECTANGLE)
{
for (unsigned i = 0; i < tc->tex_count; ++i)
@@ -532,6 +639,13 @@ opengl_fragment_shader_init_impl(opengl_tex_converter_t *tc, GLenum tex_target,
ADDF("result = color%u + result;", color_idx);
}
+ if (tc->pl_sh_res) {
+ const struct pl_shader_res *res = tc->pl_sh_res;
+ assert(res->input == PL_SHADER_SIG_COLOR);
+ assert(res->output == PL_SHADER_SIG_COLOR);
+ ADDF("result = %s(result);", res->name);
+ }
+
ADD("gl_FragColor = result * FillColor;"
"}");
diff --git a/modules/video_output/opengl/vout_helper.c b/modules/video_output/opengl/vout_helper.c
index e2491546f1..032828116b 100644
--- a/modules/video_output/opengl/vout_helper.c
+++ b/modules/video_output/opengl/vout_helper.c
@@ -566,6 +566,21 @@ opengl_deinit_program(vout_display_opengl_t *vgl, struct prgm *prgm)
vlc_object_release(tc);
if (prgm->id != 0)
vgl->vt.DeleteProgram(prgm->id);
+ FREENULL(tc->uloc.pl_vars);
+ pl_context_destroy(&tc->pl_ctx);
+}
+
+static void
+log_cb(void *priv, enum pl_log_level level, const char *msg)
+{
+ opengl_tex_converter_t *tc = priv;
+ switch (level) {
+ case PL_LOG_FATAL: // fall through
+ case PL_LOG_ERR: msg_Err(tc->gl, "%s", msg); break;
+ case PL_LOG_WARN: msg_Warn(tc->gl,"%s", msg); break;
+ case PL_LOG_INFO: msg_Info(tc->gl,"%s", msg); break;
+ default: break;
+ }
}
static int
@@ -592,6 +607,15 @@ opengl_init_program(vout_display_opengl_t *vgl, struct prgm *prgm,
#endif
tc->fmt = *fmt;
+ // create the main libplacebo context
+ tc->pl_ctx = pl_context_create(PL_API_VER, &(struct pl_context_params) {
+ .log_cb = log_cb,
+ .log_priv = tc,
+ .log_level = PL_LOG_INFO,
+ });
+ if (tc->pl_ctx)
+ tc->pl_sh = pl_shader_alloc(tc->pl_ctx, NULL, 0);
+
int ret;
if (subpics)
{
@@ -758,9 +782,13 @@ vout_display_opengl_t *vout_display_opengl_New(video_format_t *fmt,
GET_PROC_ADDR(VertexAttribPointer);
GET_PROC_ADDR(EnableVertexAttribArray);
GET_PROC_ADDR(UniformMatrix4fv);
+ GET_PROC_ADDR(UniformMatrix3fv);
+ GET_PROC_ADDR(UniformMatrix2fv);
GET_PROC_ADDR(Uniform4fv);
GET_PROC_ADDR(Uniform4f);
+ GET_PROC_ADDR(Uniform3f);
GET_PROC_ADDR(Uniform2f);
+ GET_PROC_ADDR(Uniform1f);
GET_PROC_ADDR(Uniform1i);
GET_PROC_ADDR(CreateProgram);
--
2.14.3
More information about the vlc-devel
mailing list