[vlc-devel] [PATCH] opengl: properly handle interop fmt changes
Romain Vimont
rom1v at videolabs.io
Wed Jun 24 22:55:39 CEST 2020
The interop may modify its format for two different reasons:
- to request a converter from the core
- to describe its output format to the sampler (internal changes of
software chroma, actual orientation, etc.)
The internal changes *must not* be reported to the core, to avoid an
unwanted insertion of a converter.
This was initially managed via a hack: all fields of the interop format
were copied back to the core format, except the orientation (because it
was the main "internal" change).
This hack was not sufficient due to internal chroma changes (RGB or YUV
fallbacks), which were wrongly reported to the core, leading to
unexpected insertion of converters. Commit
b79c2b224910d6987a3284ecaafeb4abb32a9a69 attempted to fix the problem by
simply not reporting any format changes to the core.
But interop_vdpau requires to change the input chroma (from
VLC_CODEC_VDPAU_VIDEO_42x to VLC_CODEC_VDPAU_OUTPUT). Since it was not
reported to the core anymore, the pictures received were not in the
expected format, causing a segfault.
To properly fix the issue, expose both an input format (reported to the
core) and an output format (for internal changes) on the interop.
This also removes the need for a "software format", which is now just a
specific property of the output format.
---
modules/hw/nvdec/nvdec_gl.c | 8 ++---
modules/video_output/opengl/interop.c | 14 ++++----
modules/video_output/opengl/interop.h | 23 +++++++++---
modules/video_output/opengl/interop_android.c | 2 +-
modules/video_output/opengl/interop_cvpx.c | 24 ++++++-------
modules/video_output/opengl/interop_dxva2.c | 36 +++++++++----------
modules/video_output/opengl/interop_sw.c | 34 +++++++++---------
modules/video_output/opengl/interop_vaapi.c | 11 +++---
modules/video_output/opengl/interop_vdpau.c | 11 +++---
modules/video_output/opengl/sampler.c | 18 +++++-----
modules/video_output/opengl/vout_helper.c | 4 +++
11 files changed, 103 insertions(+), 82 deletions(-)
diff --git a/modules/hw/nvdec/nvdec_gl.c b/modules/hw/nvdec/nvdec_gl.c
index 9d994b40cd..4ea9bf4ac1 100644
--- a/modules/hw/nvdec/nvdec_gl.c
+++ b/modules/hw/nvdec/nvdec_gl.c
@@ -133,7 +133,7 @@ tc_nvdec_gl_update(const struct vlc_gl_interop *interop, GLuint textures[],
.WidthInBytes = tex_widths[0],
.Height = tex_heights[i],
};
- if (interop->fmt.i_chroma != VLC_CODEC_NVDEC_OPAQUE && interop->fmt.i_chroma != VLC_CODEC_NVDEC_OPAQUE_444)
+ if (interop->fmt_in.i_chroma != VLC_CODEC_NVDEC_OPAQUE && interop->fmt_in.i_chroma != VLC_CODEC_NVDEC_OPAQUE_444)
cu_cpy.WidthInBytes *= 2;
result = CALL_CUDA(cuMemcpy2DAsync, &cu_cpy, 0);
if (result != VLC_SUCCESS)
@@ -156,7 +156,7 @@ static void Close(vlc_object_t *obj)
static int Open(vlc_object_t *obj)
{
struct vlc_gl_interop *interop = (void *) obj;
- if (!is_nvdec_opaque(interop->fmt.i_chroma))
+ if (!is_nvdec_opaque(interop->fmt_in.i_chroma))
return VLC_EGENERIC;
vlc_decoder_device *device = vlc_video_context_HoldDevice(interop->vctx);
@@ -199,7 +199,7 @@ static int Open(vlc_object_t *obj)
}
vlc_fourcc_t render_chroma;
- switch (interop->fmt.i_chroma)
+ switch (interop->fmt_in.i_chroma)
{
case VLC_CODEC_NVDEC_OPAQUE_10B: render_chroma = VLC_CODEC_P010; break;
case VLC_CODEC_NVDEC_OPAQUE_16B: render_chroma = VLC_CODEC_P016; break;
@@ -209,7 +209,7 @@ static int Open(vlc_object_t *obj)
default: render_chroma = VLC_CODEC_NV12; break;
}
- int ret = opengl_interop_init(interop, GL_TEXTURE_2D, render_chroma, interop->fmt.space);
+ int ret = opengl_interop_init(interop, GL_TEXTURE_2D, render_chroma, interop->fmt_in.space);
if (ret != VLC_SUCCESS)
{
vlc_decoder_device_Release(device);
diff --git a/modules/video_output/opengl/interop.c b/modules/video_output/opengl/interop.c
index 07f8c0c950..4c0252e998 100644
--- a/modules/video_output/opengl/interop.c
+++ b/modules/video_output/opengl/interop.c
@@ -39,9 +39,9 @@ vlc_gl_interop_New(struct vlc_gl_t *gl, const struct vlc_gl_api *api,
interop->init = opengl_interop_init_impl;
interop->ops = NULL;
- interop->fmt = *fmt;
+ interop->fmt_out = interop->fmt_in = *fmt;
/* this is the only allocated field, and we don't need it */
- interop->fmt.p_palette = NULL;
+ interop->fmt_out.p_palette = interop->fmt_in.p_palette = NULL;
interop->gl = gl;
interop->api = api;
@@ -95,7 +95,8 @@ vlc_gl_interop_NewForSubpictures(struct vlc_gl_t *gl,
interop->api = api;
interop->vt = &api->vt;
- video_format_Init(&interop->fmt, VLC_CODEC_RGB32);
+ video_format_Init(&interop->fmt_in, VLC_CODEC_RGB32);
+ interop->fmt_out = interop->fmt_in;
int ret = opengl_interop_generic_init(interop, false);
if (ret != VLC_SUCCESS)
@@ -355,10 +356,9 @@ opengl_interop_init_impl(struct vlc_gl_interop *interop, GLenum tex_target,
if (!desc)
return VLC_EGENERIC;
- assert(!interop->fmt.p_palette);
- interop->sw_fmt = interop->fmt;
- interop->sw_fmt.i_chroma = chroma;
- interop->sw_fmt.space = yuv_space;
+ assert(!interop->fmt_out.p_palette);
+ interop->fmt_out.i_chroma = chroma;
+ interop->fmt_out.space = yuv_space;
interop->tex_target = tex_target;
if (chroma == VLC_CODEC_XYZ12)
diff --git a/modules/video_output/opengl/interop.h b/modules/video_output/opengl/interop.h
index 9133b681ea..1f9025d5da 100644
--- a/modules/video_output/opengl/interop.h
+++ b/modules/video_output/opengl/interop.h
@@ -113,11 +113,26 @@ struct vlc_gl_interop {
const opengl_vtable_t *vt; /* for convenience, same as &api->vt */
GLenum tex_target;
- /* Can only be changed from the module open function */
- video_format_t fmt;
+ /* Input format
+ *
+ * This is the format of the pictures received from the core.
+ *
+ * It can be modified from the module open function to request changes from
+ * the core.
+ */
+ video_format_t fmt_in;
- /* Software format (useful if fmt only exposes opaque chroma) */
- video_format_t sw_fmt;
+ /* Output format
+ *
+ * This is the format of the pictures exposed by the interop to the sampler.
+ *
+ * It may differ from the input format:
+ * - the orientation may be vertically flipped
+ * - the chroma contains the "software" chroma if the input chroma is opaque
+ * - the chroma may also be changed internally to a fallback (see
+ * opengl_interop_generic_init())
+ */
+ video_format_t fmt_out;
/* Pointer to decoder video context, set by the caller (can be NULL) */
vlc_video_context *vctx;
diff --git a/modules/video_output/opengl/interop_android.c b/modules/video_output/opengl/interop_android.c
index aee9766884..8e40753acb 100644
--- a/modules/video_output/opengl/interop_android.c
+++ b/modules/video_output/opengl/interop_android.c
@@ -108,7 +108,7 @@ Open(vlc_object_t *obj)
{
struct vlc_gl_interop *interop = (void *) obj;
- if (interop->fmt.i_chroma != VLC_CODEC_ANDROID_OPAQUE
+ if (interop->fmt_in.i_chroma != VLC_CODEC_ANDROID_OPAQUE
|| !interop->gl->surface->handle.anativewindow
|| !interop->vctx)
return VLC_EGENERIC;
diff --git a/modules/video_output/opengl/interop_cvpx.c b/modules/video_output/opengl/interop_cvpx.c
index be3d9cecce..cf8b23cd23 100644
--- a/modules/video_output/opengl/interop_cvpx.c
+++ b/modules/video_output/opengl/interop_cvpx.c
@@ -166,15 +166,15 @@ Open(vlc_object_t *obj)
{
struct vlc_gl_interop *interop = (void *) obj;
- if (interop->fmt.i_chroma != VLC_CODEC_CVPX_UYVY
- && interop->fmt.i_chroma != VLC_CODEC_CVPX_NV12
- && interop->fmt.i_chroma != VLC_CODEC_CVPX_I420
- && interop->fmt.i_chroma != VLC_CODEC_CVPX_BGRA
- && interop->fmt.i_chroma != VLC_CODEC_CVPX_P010)
+ if (interop->fmt_in.i_chroma != VLC_CODEC_CVPX_UYVY
+ && interop->fmt_in.i_chroma != VLC_CODEC_CVPX_NV12
+ && interop->fmt_in.i_chroma != VLC_CODEC_CVPX_I420
+ && interop->fmt_in.i_chroma != VLC_CODEC_CVPX_BGRA
+ && interop->fmt_in.i_chroma != VLC_CODEC_CVPX_P010)
return VLC_EGENERIC;
/* The pictures are uploaded upside-down */
- video_format_TransformBy(&interop->fmt, TRANSFORM_VFLIP);
+ video_format_TransformBy(&interop->fmt_out, TRANSFORM_VFLIP);
struct priv *priv = calloc(1, sizeof(struct priv));
if (unlikely(priv == NULL))
@@ -215,10 +215,10 @@ Open(vlc_object_t *obj)
#endif
/* The pictures are uploaded upside-down */
- video_format_TransformBy(&interop->fmt, TRANSFORM_VFLIP);
+ video_format_TransformBy(&interop->fmt_out, TRANSFORM_VFLIP);
int ret;
- switch (interop->fmt.i_chroma)
+ switch (interop->fmt_in.i_chroma)
{
case VLC_CODEC_CVPX_UYVY:
/* Generate a VLC_CODEC_VYUY shader in order to use the "gbr"
@@ -228,7 +228,7 @@ Open(vlc_object_t *obj)
* extenstion. */
ret = opengl_interop_init(interop, tex_target, VLC_CODEC_VYUY,
- interop->fmt.space);
+ interop->fmt_in.space);
if (ret != VLC_SUCCESS)
goto error;
@@ -240,7 +240,7 @@ Open(vlc_object_t *obj)
case VLC_CODEC_CVPX_NV12:
{
ret = opengl_interop_init(interop, tex_target, VLC_CODEC_NV12,
- interop->fmt.space);
+ interop->fmt_in.space);
if (ret != VLC_SUCCESS)
goto error;
break;
@@ -248,7 +248,7 @@ Open(vlc_object_t *obj)
case VLC_CODEC_CVPX_P010:
{
ret = opengl_interop_init(interop, tex_target, VLC_CODEC_P010,
- interop->fmt.space);
+ interop->fmt_in.space);
if (ret != VLC_SUCCESS)
goto error;
@@ -256,7 +256,7 @@ Open(vlc_object_t *obj)
}
case VLC_CODEC_CVPX_I420:
ret = opengl_interop_init(interop, tex_target, VLC_CODEC_I420,
- interop->fmt.space);
+ interop->fmt_in.space);
if (ret != VLC_SUCCESS)
goto error;
diff --git a/modules/video_output/opengl/interop_dxva2.c b/modules/video_output/opengl/interop_dxva2.c
index 2b0f2811cb..2b32e93272 100644
--- a/modules/video_output/opengl/interop_dxva2.c
+++ b/modules/video_output/opengl/interop_dxva2.c
@@ -235,10 +235,10 @@ static void SetupProcessorInput(struct vlc_gl_interop *interop, const video_form
DXVAHD_STREAM_STATE_SOURCE_RECT_DATA srcRect;
srcRect.Enable = TRUE;
srcRect.SourceRect = (RECT) {
- .left = interop->fmt.i_x_offset,
- .right = interop->fmt.i_x_offset + interop->fmt.i_visible_width,
- .top = interop->fmt.i_y_offset,
- .bottom = interop->fmt.i_y_offset + interop->fmt.i_visible_height,
+ .left = interop->fmt_in.i_x_offset,
+ .right = interop->fmt_in.i_x_offset + interop->fmt_in.i_visible_width,
+ .top = interop->fmt_in.i_y_offset,
+ .bottom = interop->fmt_in.i_y_offset + interop->fmt_in.i_visible_height,
};;
hr = IDXVAHD_VideoProcessor_SetVideoProcessStreamState( sys->processor.proc, 0, DXVAHD_STREAM_STATE_SOURCE_RECT, sizeof(srcRect), &srcRect );
@@ -246,9 +246,9 @@ static void SetupProcessorInput(struct vlc_gl_interop *interop, const video_form
dstRect.Enable = TRUE;
dstRect.TargetRect = (RECT) {
.left = 0,
- .right = interop->fmt.i_visible_width,
+ .right = interop->fmt_in.i_visible_width,
.top = 0,
- .bottom = interop->fmt.i_visible_height,
+ .bottom = interop->fmt_in.i_visible_height,
};
hr = IDXVAHD_VideoProcessor_SetVideoProcessBltState( sys->processor.proc, DXVAHD_BLT_STATE_TARGET_RECT, sizeof(dstRect), &dstRect);
}
@@ -294,12 +294,12 @@ static int InitRangeProcessor(struct vlc_gl_interop *interop, IDirect3DDevice9Ex
DXVAHD_CONTENT_DESC desc;
desc.InputFrameFormat = DXVAHD_FRAME_FORMAT_PROGRESSIVE;
- GetFrameRate( &desc.InputFrameRate, &interop->fmt );
- desc.InputWidth = interop->fmt.i_visible_width;
- desc.InputHeight = interop->fmt.i_visible_height;
+ GetFrameRate( &desc.InputFrameRate, &interop->fmt_in );
+ desc.InputWidth = interop->fmt_in.i_visible_width;
+ desc.InputHeight = interop->fmt_in.i_visible_height;
desc.OutputFrameRate = desc.InputFrameRate;
- desc.OutputWidth = interop->fmt.i_visible_width;
- desc.OutputHeight = interop->fmt.i_visible_height;
+ desc.OutputWidth = interop->fmt_in.i_visible_width;
+ desc.OutputHeight = interop->fmt_in.i_visible_height;
hr = CreateDevice(devex, &desc, DXVAHD_DEVICE_USAGE_PLAYBACK_NORMAL, NULL, &hd_device);
if (FAILED(hr))
@@ -382,7 +382,7 @@ static int InitRangeProcessor(struct vlc_gl_interop *interop, IDirect3DDevice9Ex
}
IDXVAHD_Device_Release( hd_device );
- SetupProcessorInput(interop, &interop->fmt, src_format);
+ SetupProcessorInput(interop, &interop->fmt_in, src_format);
DXVAHD_BLT_STATE_OUTPUT_COLOR_SPACE_DATA colorspace;
colorspace.Usage = 0; // playback
@@ -407,8 +407,8 @@ GLConvOpen(vlc_object_t *obj)
{
struct vlc_gl_interop *interop = (void *) obj;
- if (interop->fmt.i_chroma != VLC_CODEC_D3D9_OPAQUE
- && interop->fmt.i_chroma != VLC_CODEC_D3D9_OPAQUE_10B)
+ if (interop->fmt_in.i_chroma != VLC_CODEC_D3D9_OPAQUE
+ && interop->fmt_in.i_chroma != VLC_CODEC_D3D9_OPAQUE_10B)
return VLC_EGENERIC;
d3d9_video_context_t *vctx_sys = GetD3D9ContextPrivate( interop->vctx );
@@ -456,7 +456,7 @@ GLConvOpen(vlc_object_t *obj)
HRESULT hr;
bool force_dxva_hd = var_InheritBool(interop, "direct3d9-dxvahd");
- if (force_dxva_hd || (interop->fmt.color_range != COLOR_RANGE_FULL &&
+ if (force_dxva_hd || (interop->fmt_in.color_range != COLOR_RANGE_FULL &&
d3d9_decoder->d3ddev.identifier.VendorId == GPU_MANUFACTURER_NVIDIA))
{
// NVIDIA bug, YUV to RGB internal conversion in StretchRect always converts from limited to limited range
@@ -482,8 +482,8 @@ GLConvOpen(vlc_object_t *obj)
HANDLE shared_handle = NULL;
hr = IDirect3DDevice9Ex_CreateRenderTarget(d3d9_decoder->d3ddev.devex,
- interop->fmt.i_visible_width,
- interop->fmt.i_visible_height,
+ interop->fmt_in.i_visible_width,
+ interop->fmt_in.i_visible_height,
priv->OutputFormat,
D3DMULTISAMPLE_NONE, 0, FALSE,
&priv->dx_render, &shared_handle);
@@ -510,7 +510,7 @@ GLConvOpen(vlc_object_t *obj)
interop->ops = &ops;
/* The pictures are uploaded upside-down */
- video_format_TransformBy(&interop->fmt, TRANSFORM_VFLIP);
+ video_format_TransformBy(&interop->fmt_out, TRANSFORM_VFLIP);
int ret = opengl_interop_init(interop, GL_TEXTURE_2D, VLC_CODEC_RGB32,
COLOR_SPACE_UNDEF);
diff --git a/modules/video_output/opengl/interop_sw.c b/modules/video_output/opengl/interop_sw.c
index 5571d6bcef..486b2b3f1d 100644
--- a/modules/video_output/opengl/interop_sw.c
+++ b/modules/video_output/opengl/interop_sw.c
@@ -70,7 +70,7 @@ pbo_picture_create(const struct vlc_gl_interop *interop)
.p_sys = picsys,
.pf_destroy = pbo_picture_destroy,
};
- picture_t *pic = picture_NewFromResource(&interop->fmt, &rsc);
+ picture_t *pic = picture_NewFromResource(&interop->fmt_out, &rsc);
if (pic == NULL)
{
free(picsys);
@@ -81,7 +81,7 @@ pbo_picture_create(const struct vlc_gl_interop *interop)
picsys->DeleteBuffers = interop->vt->DeleteBuffers;
/* XXX: needed since picture_NewFromResource override pic planes */
- if (picture_Setup(pic, &interop->fmt))
+ if (picture_Setup(pic, &interop->fmt_out))
{
picture_Release(pic);
return NULL;
@@ -299,17 +299,17 @@ opengl_interop_generic_init(struct vlc_gl_interop *interop, bool allow_dr)
video_color_space_t space;
const vlc_fourcc_t *list;
- if (vlc_fourcc_IsYUV(interop->fmt.i_chroma))
+ if (vlc_fourcc_IsYUV(interop->fmt_in.i_chroma))
{
GLint max_texture_units = 0;
interop->vt->GetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &max_texture_units);
if (max_texture_units < 3)
return VLC_EGENERIC;
- list = vlc_fourcc_GetYUVFallback(interop->fmt.i_chroma);
- space = interop->fmt.space;
+ list = vlc_fourcc_GetYUVFallback(interop->fmt_in.i_chroma);
+ space = interop->fmt_in.space;
}
- else if (interop->fmt.i_chroma == VLC_CODEC_XYZ12)
+ else if (interop->fmt_in.i_chroma == VLC_CODEC_XYZ12)
{
static const vlc_fourcc_t xyz12_list[] = { VLC_CODEC_XYZ12, 0 };
list = xyz12_list;
@@ -317,12 +317,12 @@ opengl_interop_generic_init(struct vlc_gl_interop *interop, bool allow_dr)
}
else
{
- list = vlc_fourcc_GetRGBFallback(interop->fmt.i_chroma);
+ list = vlc_fourcc_GetRGBFallback(interop->fmt_in.i_chroma);
space = COLOR_SPACE_UNDEF;
}
/* The pictures are uploaded upside-down */
- video_format_TransformBy(&interop->fmt, TRANSFORM_VFLIP);
+ video_format_TransformBy(&interop->fmt_out, TRANSFORM_VFLIP);
int ret = VLC_EGENERIC;
while (*list)
@@ -330,20 +330,20 @@ opengl_interop_generic_init(struct vlc_gl_interop *interop, bool allow_dr)
ret = opengl_interop_init(interop, GL_TEXTURE_2D, *list, space);
if (ret == VLC_SUCCESS)
{
- interop->fmt.i_chroma = *list;
+ interop->fmt_out.i_chroma = *list;
- if (interop->fmt.i_chroma == VLC_CODEC_RGB32)
+ if (interop->fmt_out.i_chroma == VLC_CODEC_RGB32)
{
#if defined(WORDS_BIGENDIAN)
- interop->fmt.i_rmask = 0xff000000;
- interop->fmt.i_gmask = 0x00ff0000;
- interop->fmt.i_bmask = 0x0000ff00;
+ interop->fmt_out.i_rmask = 0xff000000;
+ interop->fmt_out.i_gmask = 0x00ff0000;
+ interop->fmt_out.i_bmask = 0x0000ff00;
#else
- interop->fmt.i_rmask = 0x000000ff;
- interop->fmt.i_gmask = 0x0000ff00;
- interop->fmt.i_bmask = 0x00ff0000;
+ interop->fmt_out.i_rmask = 0x000000ff;
+ interop->fmt_out.i_gmask = 0x0000ff00;
+ interop->fmt_out.i_bmask = 0x00ff0000;
#endif
- video_format_FixRgb(&interop->fmt);
+ video_format_FixRgb(&interop->fmt_out);
}
break;
}
diff --git a/modules/video_output/opengl/interop_vaapi.c b/modules/video_output/opengl/interop_vaapi.c
index b1ed1bd95a..c7646eaed3 100644
--- a/modules/video_output/opengl/interop_vaapi.c
+++ b/modules/video_output/opengl/interop_vaapi.c
@@ -356,7 +356,8 @@ tc_va_check_derive_image(const struct vlc_gl_interop *interop)
VASurfaceID *va_surface_ids;
picture_pool_t *pool = vlc_vaapi_PoolNew(o, interop->vctx, priv->vadpy, 1,
- &va_surface_ids, &interop->fmt);
+ &va_surface_ids,
+ &interop->fmt_out);
if (!pool)
return VLC_EGENERIC;
@@ -422,7 +423,7 @@ Open(vlc_object_t *obj)
return VLC_EGENERIC;
vlc_decoder_device *dec_device = vlc_video_context_HoldDevice(interop->vctx);
if (dec_device->type != VLC_DECODER_DEVICE_VAAPI
- || !vlc_vaapi_IsChromaOpaque(interop->fmt.i_chroma)
+ || !vlc_vaapi_IsChromaOpaque(interop->fmt_in.i_chroma)
|| interop->gl->ext != VLC_GL_EXT_EGL
|| interop->gl->egl.createImageKHR == NULL
|| interop->gl->egl.destroyImageKHR == NULL)
@@ -451,7 +452,7 @@ Open(vlc_object_t *obj)
int va_fourcc;
int vlc_sw_chroma;
- switch (interop->fmt.i_chroma)
+ switch (interop->fmt_in.i_chroma)
{
case VLC_CODEC_VAAPI_420:
va_fourcc = VA_FOURCC_NV12;
@@ -483,10 +484,10 @@ Open(vlc_object_t *obj)
goto error;
/* The pictures are uploaded upside-down */
- video_format_TransformBy(&interop->fmt, TRANSFORM_VFLIP);
+ video_format_TransformBy(&interop->fmt_out, TRANSFORM_VFLIP);
int ret = opengl_interop_init(interop, GL_TEXTURE_2D, vlc_sw_chroma,
- interop->fmt.space);
+ interop->fmt_in.space);
if (ret != VLC_SUCCESS)
goto error;
diff --git a/modules/video_output/opengl/interop_vdpau.c b/modules/video_output/opengl/interop_vdpau.c
index 9b0d3adf31..9919630ed7 100644
--- a/modules/video_output/opengl/interop_vdpau.c
+++ b/modules/video_output/opengl/interop_vdpau.c
@@ -122,9 +122,9 @@ Open(vlc_object_t *obj)
return VLC_EGENERIC;
vlc_decoder_device *dec_device = vlc_video_context_HoldDevice(interop->vctx);
if (GetVDPAUOpaqueDevice(dec_device) == NULL
- || (interop->fmt.i_chroma != VLC_CODEC_VDPAU_VIDEO_420
- && interop->fmt.i_chroma != VLC_CODEC_VDPAU_VIDEO_422
- && interop->fmt.i_chroma != VLC_CODEC_VDPAU_VIDEO_444)
+ || (interop->fmt_in.i_chroma != VLC_CODEC_VDPAU_VIDEO_420
+ && interop->fmt_in.i_chroma != VLC_CODEC_VDPAU_VIDEO_422
+ && interop->fmt_in.i_chroma != VLC_CODEC_VDPAU_VIDEO_444)
|| !vlc_gl_StrHasToken(interop->api->extensions, "GL_NV_vdpau_interop")
|| interop->gl->surface->type != VOUT_WINDOW_TYPE_XID)
{
@@ -140,7 +140,8 @@ Open(vlc_object_t *obj)
}
sys->dec_device = dec_device;
- interop->fmt.i_chroma = VLC_CODEC_VDPAU_OUTPUT;
+ /* Request to change the input chroma to the core */
+ interop->fmt_in.i_chroma = VLC_CODEC_VDPAU_OUTPUT;
VdpDevice device;
vdpau_decoder_device_t *vdpau_dev = GetVDPAUOpaqueDevice(dec_device);
@@ -177,7 +178,7 @@ Open(vlc_object_t *obj)
INTEROP_CALL(glVDPAUInitNV, (void *)(uintptr_t)device, vdp_gpa);
/* The pictures are uploaded upside-down */
- video_format_TransformBy(&interop->fmt, TRANSFORM_VFLIP);
+ video_format_TransformBy(&interop->fmt_out, TRANSFORM_VFLIP);
int ret = opengl_interop_init(interop, GL_TEXTURE_2D, VLC_CODEC_RGB32,
COLOR_SPACE_UNDEF);
diff --git a/modules/video_output/opengl/sampler.c b/modules/video_output/opengl/sampler.c
index 7528d88135..4cfeada6ba 100644
--- a/modules/video_output/opengl/sampler.c
+++ b/modules/video_output/opengl/sampler.c
@@ -783,7 +783,7 @@ opengl_fragment_shader_init(struct vlc_gl_sampler *sampler, GLenum tex_target,
dst_space.transfer = var_InheritInteger(priv->gl, "target-trc");
pl_shader_color_map(sh, &color_params,
- vlc_placebo_ColorSpace(&interop->fmt),
+ vlc_placebo_ColorSpace(&interop->fmt_out),
dst_space, NULL, false);
struct pl_shader_obj *dither_state = NULL;
@@ -834,8 +834,8 @@ opengl_fragment_shader_init(struct vlc_gl_sampler *sampler, GLenum tex_target,
ADD(res->glsl);
}
#else
- if (interop->fmt.transfer == TRANSFER_FUNC_SMPTE_ST2084 ||
- interop->fmt.primaries == COLOR_PRIMARIES_BT2020)
+ if (interop->fmt_out.transfer == TRANSFER_FUNC_SMPTE_ST2084 ||
+ interop->fmt_out.primaries == COLOR_PRIMARIES_BT2020)
{
// no warning for HLG because it's more or less backwards-compatible
msg_Warn(priv->gl, "VLC needs to be built with support for libplacebo "
@@ -953,7 +953,7 @@ vlc_gl_sampler_New(struct vlc_gl_interop *interop)
priv->gl = interop->gl;
priv->vt = interop->vt;
- sampler->fmt = &interop->sw_fmt;
+ sampler->fmt = &interop->fmt_out;
sampler->shader.extensions = NULL;
sampler->shader.body = NULL;
@@ -983,9 +983,9 @@ vlc_gl_sampler_New(struct vlc_gl_interop *interop)
int ret =
opengl_fragment_shader_init(sampler, interop->tex_target,
- interop->sw_fmt.i_chroma,
- interop->sw_fmt.space,
- interop->sw_fmt.orientation);
+ interop->fmt_out.i_chroma,
+ interop->fmt_out.space,
+ interop->fmt_out.orientation);
if (ret != VLC_SUCCESS)
{
free(sampler);
@@ -994,9 +994,9 @@ vlc_gl_sampler_New(struct vlc_gl_interop *interop)
/* Texture size */
for (unsigned j = 0; j < interop->tex_count; j++) {
- const GLsizei w = interop->fmt.i_visible_width * interop->texs[j].w.num
+ const GLsizei w = interop->fmt_out.i_visible_width * interop->texs[j].w.num
/ interop->texs[j].w.den;
- const GLsizei h = interop->fmt.i_visible_height * interop->texs[j].h.num
+ const GLsizei h = interop->fmt_out.i_visible_height * interop->texs[j].h.num
/ interop->texs[j].h.den;
if (interop->api->supports_npot) {
priv->tex_widths[j] = w;
diff --git a/modules/video_output/opengl/vout_helper.c b/modules/video_output/opengl/vout_helper.c
index 526edcb23f..b6105dd062 100644
--- a/modules/video_output/opengl/vout_helper.c
+++ b/modules/video_output/opengl/vout_helper.c
@@ -178,6 +178,10 @@ vout_display_opengl_t *vout_display_opengl_New(video_format_t *fmt,
&& vout_display_opengl_SetViewpoint(vgl, viewpoint) != VLC_SUCCESS)
goto delete_sub_renderer;
+ /* Forward to the core the changes to the input format requested by the
+ * interop */
+ *fmt = vgl->interop->fmt_in;
+
if (subpicture_chromas) {
*subpicture_chromas = gl_subpicture_chromas;
}
--
2.27.0
More information about the vlc-devel
mailing list