[vlc-devel] [PATCH] vout: opengl: add NV12/NV12 support
Rémi Denis-Courmont
remi at remlab.net
Sat Dec 3 13:10:16 CET 2016
On December 1, 2016 8:15:52 AM EST, Thomas Guillem <thomas at gllm.fr> wrote:
>---
>modules/video_output/opengl.c | 152
>+++++++++++++++++++++++++++++++++---------
> 1 file changed, 121 insertions(+), 31 deletions(-)
>
>diff --git a/modules/video_output/opengl.c
>b/modules/video_output/opengl.c
>index 2b994bd..7ae7a92 100644
>--- a/modules/video_output/opengl.c
>+++ b/modules/video_output/opengl.c
>@@ -104,6 +104,9 @@
> #ifndef GL_R16
> #define GL_R16 0
> #endif
>+#ifndef GL_RG
>+#define GL_RG 0
>+#endif
>
> #define SPHERE_RADIUS 1.f
>
>@@ -131,10 +134,11 @@ struct vout_display_opengl_t {
>
> video_format_t fmt;
> const vlc_chroma_description_t *chroma;
>+ unsigned samples_per_pixel[PICTURE_PLANE_MAX];
>
> int tex_target;
>- int tex_format;
>- int tex_internal;
>+ int tex_format[PICTURE_PLANE_MAX];
>+ int tex_internal[PICTURE_PLANE_MAX];
> int tex_type;
>
> int tex_width[PICTURE_PLANE_MAX];
>@@ -322,7 +326,7 @@ static void
>BuildYUVFragmentShader(vout_display_opengl_t *vgl,
> };
>
> /* Basic linear YUV -> RGB conversion using bilinear interpolation */
>- const char *template_glsl_yuv =
>+ static const char *template_glsl_yuv3 =
> "#version " GLSL_VERSION "\n"
> PRECISION
> "uniform sampler2D Texture0;"
>@@ -350,14 +354,54 @@ static void
>BuildYUVFragmentShader(vout_display_opengl_t *vgl,
> " result = (z * Coefficient[2]) + result;"
> " gl_FragColor = result;"
> "}";
>- bool swap_uv = fmt->i_chroma == VLC_CODEC_YV12 ||
>- fmt->i_chroma == VLC_CODEC_YV9;
>+ static const char *template_glsl_yuv2 =
>+ "#version " GLSL_VERSION "\n"
>+ PRECISION
>+ "uniform sampler2D Texture0;"
>+ "uniform sampler2D Texture1;"
>+ "uniform vec4 Coefficient[4];"
>+ "varying vec4 TexCoord0,TexCoord1;"
>+
>+ "void main(void) {"
>+ " vec4 x,y,z,result;"
>+
>+ " float val0 = texture2D(Texture0, TexCoord0.st).x;"
>+ " float val1 = texture2D(Texture1, TexCoord1.st).x;"
>+ /* The second sample on the second texture is either on 'g'
>(for GL_RG)
>+ * or on 'a' (for GL_LUMINANCE_ALPHA) */
>+ " float val2 = texture2D(Texture1, TexCoord1.st).%c;"
>+ " x = vec4(val0, val0, val0, 1);"
>+ " %c = vec4(val1, val1, val1, 1);"
>+ " %c = vec4(val2, val2, val2, 1l);"
>+
>+ " result = x * Coefficient[0] + Coefficient[3];"
>+ " result = (y * Coefficient[1]) + result;"
>+ " result = (z * Coefficient[2]) + result;"
>+ " gl_FragColor = result;"
>+ "}";
>
> char *code;
>- if (asprintf(&code, template_glsl_yuv,
>- swap_uv ? 'z' : 'y',
>- swap_uv ? 'y' : 'z') < 0)
>- return;
>+ if (vgl->chroma->plane_count == 3)
>+ {
>+ bool swap_uv = fmt->i_chroma == VLC_CODEC_YV12 ||
>+ fmt->i_chroma == VLC_CODEC_YV9;
>+ if (asprintf(&code, template_glsl_yuv3,
>+ swap_uv ? 'z' : 'y',
>+ swap_uv ? 'y' : 'z') < 0)
>+ return;
>+ }
>+ else
>+ {
>+ assert(vgl->tex_format[0] == GL_RED || vgl->tex_format[0] ==
>GL_LUMINANCE);
>+ assert(vgl->tex_format[1] == GL_RG || vgl->tex_format[1] ==
>GL_LUMINANCE_ALPHA);
>+
>+ bool swap_uv = fmt->i_chroma == VLC_CODEC_NV21;
>+ if (asprintf(&code, template_glsl_yuv2,
>+ vgl->tex_format[1] == GL_RG ? 'g' : 'a',
>+ swap_uv ? 'z' : 'y',
>+ swap_uv ? 'y' : 'z') < 0)
>+ return;
>+ }
>
> for (int i = 0; i < 4; i++) {
> float correction = i < 3 ? yuv_range_correction : 1.f;
>@@ -485,9 +529,11 @@ vout_display_opengl_t
>*vout_display_opengl_New(video_format_t *fmt,
> const bool oglv3 = strverscmp((const char *)ogl_version, "3.0") >= 0;
> const int yuv_plane_texformat = oglv3 ? GL_RED : GL_LUMINANCE;
> const int yuv_plane_texformat_16 = oglv3 ? GL_R16 : GL_LUMINANCE16;
>+ const int yuv_plane_texformat_2samples = oglv3 ? GL_RG :
>GL_LUMINANCE_ALPHA;
> #else
> bool supports_shaders = false;
> const int yuv_plane_texformat = GL_LUMINANCE;
>+ const int yuv_plane_texformat_2samples = GL_LUMINANCE_ALPHA;
> #endif
>
> #if USE_OPENGL_ES == 2
>@@ -596,10 +642,11 @@ vout_display_opengl_t
>*vout_display_opengl_New(video_format_t *fmt,
> vgl->fmt.i_gmask = 0x0000ff00;
> vgl->fmt.i_bmask = 0x00ff0000;
> # endif
>- vgl->tex_target = GL_TEXTURE_2D;
>- vgl->tex_format = GL_RGBA;
>- vgl->tex_internal = GL_RGBA;
>- vgl->tex_type = GL_UNSIGNED_BYTE;
>+ vgl->tex_target = GL_TEXTURE_2D;
>+ vgl->tex_format[0] = GL_RGBA;
>+ vgl->tex_internal[0] = GL_RGBA;
>+ vgl->samples_per_pixel[0] = 1;
>+ vgl->tex_type = GL_UNSIGNED_BYTE;
> /* Use YUV if possible and needed */
> bool need_fs_yuv = false;
> bool need_fs_xyz = false;
>@@ -607,16 +654,41 @@ vout_display_opengl_t
>*vout_display_opengl_New(video_format_t *fmt,
> bool need_vs = fmt->projection_mode != PROJECTION_MODE_RECTANGULAR;
> float yuv_range_correction = 1.0;
>
>- if (max_texture_units >= 3 && supports_shaders &&
>vlc_fourcc_IsYUV(fmt->i_chroma)) {
>+ if (max_texture_units >= 2 && supports_shaders &&
>vlc_fourcc_IsYUV(fmt->i_chroma)) {
> const vlc_fourcc_t *list = vlc_fourcc_GetYUVFallback(fmt->i_chroma);
> while (*list) {
>const vlc_chroma_description_t *dsc =
>vlc_fourcc_GetChromaDescription(*list);
>- if (dsc && dsc->plane_count == 3 && dsc->pixel_size == 1)
>{
>+ if (!dsc)
>+ continue;
>+ if ((unsigned) max_texture_units >= dsc->plane_count
>+ && dsc->plane_count <= 3 && dsc->pixel_size == 1) {
>+ /* Accept yuv format with 2 or 3 planes if there are
>enough
>+ * textures */
>+
>+ if (dsc->plane_count == 2)
>+ {
>+ if (*list != VLC_CODEC_NV12 && *list !=
>VLC_CODEC_NV21)
>+ continue;
>+ vgl->samples_per_pixel[0] = 1;
>+ vgl->samples_per_pixel[1] = 2;
>+ vgl->tex_format[0] = yuv_plane_texformat;
>+ vgl->tex_internal[0] = yuv_plane_texformat;
>+ vgl->tex_format[1] =
>yuv_plane_texformat_2samples;
>+ vgl->tex_internal[1] =
>yuv_plane_texformat_2samples;
>+ }
>+ else
>+ {
>+ for (unsigned i = 0; i < dsc->plane_count; ++i)
>+ {
>+ vgl->tex_format[i] = yuv_plane_texformat;
>+ vgl->tex_internal[i] = yuv_plane_texformat;
>+ vgl->samples_per_pixel[i] = 1;
>+ }
>+ }
>+
> need_fs_yuv = true;
> vgl->fmt = *fmt;
> vgl->fmt.i_chroma = *list;
>- vgl->tex_format = yuv_plane_texformat;
>- vgl->tex_internal = yuv_plane_texformat;
> vgl->tex_type = GL_UNSIGNED_BYTE;
> yuv_range_correction = 1.0;
> break;
>@@ -629,9 +701,13 @@ vout_display_opengl_t
>*vout_display_opengl_New(video_format_t *fmt,
> need_fs_yuv = true;
> vgl->fmt = *fmt;
> vgl->fmt.i_chroma = *list;
>- vgl->tex_format = yuv_plane_texformat;
>- vgl->tex_internal = yuv_plane_texformat_16;
> vgl->tex_type = GL_UNSIGNED_SHORT;
>+ for (unsigned i = 0; i < dsc->plane_count; ++i)
>+ {
>+ vgl->tex_format[i] = yuv_plane_texformat;
>+ vgl->tex_internal[i] = yuv_plane_texformat_16;
>+ vgl->samples_per_pixel[i] = 1;
>+ }
>yuv_range_correction = (float)((1 << 16) - 1) / ((1 << dsc->pixel_bits)
>- 1);
> break;
> #endif
>@@ -639,23 +715,26 @@ vout_display_opengl_t
>*vout_display_opengl_New(video_format_t *fmt,
> list++;
> }
> }
>-
> if (fmt->i_chroma == VLC_CODEC_XYZ12) {
> need_fs_xyz = true;
> vgl->fmt = *fmt;
> vgl->fmt.i_chroma = VLC_CODEC_XYZ12;
>- vgl->tex_format = GL_RGB;
>- vgl->tex_internal = GL_RGB;
> vgl->tex_type = GL_UNSIGNED_SHORT;
>+ vgl->tex_format[0] = GL_RGB;
>+ vgl->tex_internal[0] = GL_RGB;
>+ vgl->samples_per_pixel[0] = 1;
> }
>+
> vgl->chroma = vlc_fourcc_GetChromaDescription(vgl->fmt.i_chroma);
> assert(vgl->chroma != NULL);
> vgl->use_multitexture = vgl->chroma->plane_count > 1;
>
> /* Texture size */
> for (unsigned j = 0; j < vgl->chroma->plane_count; j++) {
>- int w = vgl->fmt.i_visible_width * vgl->chroma->p[j].w.num /
>vgl->chroma->p[j].w.den;
>- int h = vgl->fmt.i_visible_height * vgl->chroma->p[j].h.num /
>vgl->chroma->p[j].h.den;
>+ int w = vgl->fmt.i_visible_width * vgl->chroma->p[j].w.num
>+ / vgl->chroma->p[j].w.den / vgl->samples_per_pixel[j];
>+ int h = vgl->fmt.i_visible_height * vgl->chroma->p[j].h.num
>+ / vgl->chroma->p[j].h.den;
> if (vgl->supports_npot) {
> vgl->tex_width[j] = w;
> vgl->tex_height[j] = h;
>@@ -937,9 +1016,9 @@ picture_pool_t
>*vout_display_opengl_GetPool(vout_display_opengl_t *vgl, unsigned
> glTexParameteri(vgl->tex_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
>
> /* Call glTexImage2D only once, and use glTexSubImage2D later */
>- glTexImage2D(vgl->tex_target, 0,
>- vgl->tex_internal, vgl->tex_width[j],
>vgl->tex_height[j],
>- 0, vgl->tex_format, vgl->tex_type, NULL);
>+ glTexImage2D(vgl->tex_target, 0, vgl->tex_internal[j],
>+ vgl->tex_width[j], vgl->tex_height[j], 0,
>+ vgl->tex_format[j], vgl->tex_type, NULL);
> }
> }
>
>@@ -1034,8 +1113,13 @@ int
>vout_display_opengl_Prepare(vout_display_opengl_t *vgl,
>
>Upload(vgl, picture->format.i_visible_width, vgl->fmt.i_visible_height,
> vgl->fmt.i_width, vgl->fmt.i_height,
>- vgl->chroma->p[j].w.num, vgl->chroma->p[j].w.den,
>vgl->chroma->p[j].h.num, vgl->chroma->p[j].h.den,
>- picture->p[j].i_pitch, picture->p[j].i_pixel_pitch, 0,
>picture->p[j].p_pixels, vgl->tex_target, vgl->tex_format,
>vgl->tex_type);
>+ vgl->chroma->p[j].w.num,
>+ vgl->chroma->p[j].w.den * vgl->samples_per_pixel[j],
>+ vgl->chroma->p[j].h.num, vgl->chroma->p[j].h.den,
>+ picture->p[j].i_pitch,
>+ picture->p[j].i_pixel_pitch *
>vgl->samples_per_pixel[j], 0,
>+ picture->p[j].p_pixels, vgl->tex_target,
>vgl->tex_format[j],
>+ vgl->tex_type);
> }
>
> int last_count = vgl->region_count;
>@@ -1617,6 +1701,10 @@ static void
>DrawWithShaders(vout_display_opengl_t *vgl,
>vgl->Uniform1i(vgl->GetUniformLocation(vgl->program[0], "Texture0"),
>0);
>vgl->Uniform1i(vgl->GetUniformLocation(vgl->program[0], "Texture1"),
>1);
>vgl->Uniform1i(vgl->GetUniformLocation(vgl->program[0], "Texture2"),
>2);
>+ } else if (vgl->chroma->plane_count == 2) {
>+ vgl->Uniform4fv(vgl->GetUniformLocation(vgl->program[0],
>"Coefficient"), 4, vgl->local_value);
>+ vgl->Uniform1i(vgl->GetUniformLocation(vgl->program[0],
>"Texture0"), 0);
>+ vgl->Uniform1i(vgl->GetUniformLocation(vgl->program[0],
>"Texture1"), 1);
> }
> else if (vgl->chroma->plane_count == 1) {
>vgl->Uniform1i(vgl->GetUniformLocation(vgl->program[0], "Texture0"),
>0);
>@@ -1747,8 +1835,10 @@ int
>vout_display_opengl_Display(vout_display_opengl_t *vgl,
> float scale_w, scale_h;
>
> if (vgl->tex_target == GL_TEXTURE_2D) {
>- scale_w = (float)vgl->chroma->p[j].w.num /
>vgl->chroma->p[j].w.den / vgl->tex_width[j];
>- scale_h = (float)vgl->chroma->p[j].h.num /
>vgl->chroma->p[j].h.den / vgl->tex_height[j];
>+ scale_w = (float)vgl->chroma->p[j].w.num /
>vgl->chroma->p[j].w.den
>+ / vgl->samples_per_pixel[j] / vgl->tex_width[j];
>+ scale_h = (float)vgl->chroma->p[j].h.num /
>vgl->chroma->p[j].h.den
>+ / vgl->tex_height[j];
>
> } else {
> scale_w = 1.0;
>--
>2.10.2
>
>_______________________________________________
>vlc-devel mailing list
>To unsubscribe or modify your subscription options:
>https://mailman.videolan.org/listinfo/vlc-devel
Hi,
I have not had time to read my OpenGL paper book, so I can't comment on the OpenGL aspects. But I think that the file is getting much too big, but I don't like meaningless splits with tight dependencies between files either :/
I would though think that chroma conversion belongs in a separate conversion plugin. It might not be feasible without an opaque OpenGL texture chroma type however.
Also, NV12 specifically is a hardware decoder output format. For sure, it needs to be supported eventually... But does it really make sense at this point, that the OpenGL output does not support pass-through, so the NV12 will be copied anyway. If copied, then conversion to I420 can be done on the fly for free in SIMD. IIRC, Laurent even implemented that for x86 already.
--
Rémi Denis-Courmont
More information about the vlc-devel
mailing list