[vlc-devel] [PATCH 3/4] qsv: add support for encoding from D3D11 source textures
Steve Lhomme
robux4 at ycbcr.xyz
Thu Jul 2 16:18:59 CEST 2020
---
modules/codec/Makefile.am | 3 +
modules/codec/qsv.c | 329 +++++++++++++++++++++++++++++++++++++-
2 files changed, 324 insertions(+), 8 deletions(-)
diff --git a/modules/codec/Makefile.am b/modules/codec/Makefile.am
index dc0bda07dc8..9dcd816701a 100644
--- a/modules/codec/Makefile.am
+++ b/modules/codec/Makefile.am
@@ -568,6 +568,9 @@ libqsv_plugin_la_SOURCES = codec/qsv.c codec/vlc_fifo_helper.h demux/mpeg/timest
libqsv_plugin_la_CFLAGS = $(AM_CFLAGS) $(MFX_CFLAGS)
libqsv_plugin_la_LDFLAGS = $(AM_LDFLAGS) -rpath '$(codecdir)'
libqsv_plugin_la_LIBADD = $(MFX_LIBS)
+if HAVE_AVCODEC_D3D11VA
+libqsv_plugin_la_LIBADD += libd3d11_common.la $(LIBCOM) -luuid
+endif
EXTRA_LTLIBRARIES += libqsv_plugin.la
codec_LTLIBRARIES += $(LTLIBqsv)
diff --git a/modules/codec/qsv.c b/modules/codec/qsv.c
index b04c1dcb899..c7abb37d557 100644
--- a/modules/codec/qsv.c
+++ b/modules/codec/qsv.c
@@ -38,6 +38,13 @@
#include <mfx/mfxvideo.h>
#include "../demux/mpeg/timestamps.h"
+#if HAVE_AVCODEC_D3D11VA
+#define COBJMACROS
+#include <initguid.h>
+#include <d3d11.h>
+#include "../video_chroma/d3d11_fmt.h"
+#endif
+
#define SOUT_CFG_PREFIX "sout-qsv-"
#define QSV_HAVE_CO2 (MFX_VERSION_MAJOR > 1 || (MFX_VERSION_MAJOR == 1 && MFX_VERSION_MINOR >= 6))
@@ -268,6 +275,7 @@ typedef struct _QSVFrame
mfxFrameSurface1 surface;
mfxEncodeCtrl enc_ctrl;
int used;
+ int mem_type;
} QSVFrame;
typedef struct async_task_t
@@ -295,6 +303,13 @@ typedef struct
picture_pool_t *input_pool; // pool of pictures to feed the decoder
// as it doesn't like constantly changing buffers
vlc_decoder_device *dec_device;
+#if HAVE_AVCODEC_D3D11VA
+ size_t numAllocated;
+ QSVFrame *allocatedFrames[16];
+
+ /* Direct3D11 */
+ ID3D11Texture2D *copy_pool[64]; /* shadow pool because it needs textures with only one slice */
+#endif
} encoder_sys_t;
static block_t *Encode(encoder_t *, picture_t *);
@@ -369,6 +384,233 @@ static uint64_t qsv_params_get_value(const char *const *text,
return list[result];
}
+#if HAVE_AVCODEC_D3D11VA
+static const d3d_format_t *GetTextureFormat(DXGI_FORMAT format, vlc_fourcc_t fcc)
+{
+ for (const d3d_format_t *output_format = GetRenderFormatList();
+ output_format->name != NULL; ++output_format)
+ {
+ if ( output_format->formatTexture == format &&
+ (!fcc || fcc == output_format->fourcc) )
+ {
+ return output_format;
+ }
+ }
+ return NULL;
+}
+
+static inline mfxMemId qsvfMemId(QSVFrame *qf)
+{
+ return (mfxMemId)qf;
+}
+
+static inline QSVFrame *memIdQsv(mfxMemId mId)
+{
+ return (QSVFrame*)mId;
+}
+
+static mfxStatus qsv_frame_alloc(mfxHDL pthis, mfxFrameAllocRequest *request, mfxFrameAllocResponse *response)
+{
+ if (request->Type & MFX_MEMTYPE_SYSTEM_MEMORY)
+ return MFX_ERR_MEMORY_ALLOC;
+
+ encoder_t *enc = pthis;
+ encoder_sys_t *p_sys = enc->p_sys;
+ mfxMemId *mids = calloc(request->NumFrameSuggested, sizeof(mfxMemId));
+ if (unlikely(mids == NULL))
+ return MFX_ERR_MEMORY_ALLOC;
+
+ d3d11_decoder_device_t *d3d11_dev = GetD3D11OpaqueDevice(p_sys->dec_device);
+
+ for (size_t i=0; i < request->NumFrameSuggested &&
+ (i + request->NumFrameSuggested) < ARRAYSIZE(p_sys->allocatedFrames); i++)
+ {
+ QSVFrame *qf = calloc(1,sizeof(QSVFrame));
+
+ video_format_t surface_fmt = enc->fmt_in.video;
+ surface_fmt.i_chroma = VLC_CODEC_D3D11_OPAQUE;
+ surface_fmt.i_bits_per_pixel = 12;
+ surface_fmt.i_width = request->Info.Width;
+ surface_fmt.i_height = request->Info.Height;
+
+ if (request->Info.FourCC == MFX_FOURCC_NV12)
+ qf->pic = D3D11_AllocPicture(VLC_OBJECT(enc), &surface_fmt, NULL/*TODO*/, GetTextureFormat(DXGI_FORMAT_NV12, VLC_CODEC_NV12));
+ else if (request->Info.FourCC == MFX_FOURCC_P8)
+ {
+ D3D11_BUFFER_DESC desc = { 0 };
+
+ desc.ByteWidth = request->Info.Width * request->Info.Height;
+ desc.Usage = D3D11_USAGE_STAGING;
+ desc.BindFlags = 0;
+ desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
+ desc.MiscFlags = 0;
+ desc.StructureByteStride = 0;
+
+ ID3D11Buffer *buffer = NULL;
+ HRESULT hr = ID3D11Device_CreateBuffer(d3d11_dev->d3d_dev.d3ddevice, &desc, 0, &buffer);
+ if (FAILED(hr))
+ {
+ for (;;)
+ {
+ qf = p_sys->allocatedFrames[--i + p_sys->numAllocated];
+ ID3D11Buffer *buffer = qf->surface.Data.MemId;
+ ID3D11Buffer_Release(buffer);
+ if (i == 0)
+ {
+ free(mids);
+ return MFX_ERR_NOT_ENOUGH_BUFFER;
+ }
+ }
+ }
+ qf->surface.Data.MemId = buffer;
+ }
+ else
+ assert(0); /* not supported yet */
+
+ qf->mem_type = request->Type;
+ p_sys->allocatedFrames[i + p_sys->numAllocated] = qf;
+ mids[i] = qsvfMemId(qf);
+ }
+ response->mids = mids;
+ response->NumFrameActual = request->NumFrameSuggested;
+ p_sys->numAllocated += request->NumFrameSuggested;
+ return MFX_ERR_NONE;
+}
+
+static mfxStatus qsv_frame_free(mfxHDL pthis, mfxFrameAllocResponse *response)
+{
+ encoder_t *enc = pthis;
+ encoder_sys_t *p_sys = enc->p_sys;
+ mfxMemId *mids = response->mids;
+ QSVFrame *qf = memIdQsv(mids[0]);
+ for (size_t i = 0; i < p_sys->numAllocated; i++)
+ {
+ qf = p_sys->allocatedFrames[i];
+ if (qf == memIdQsv(mids[0]))
+ {
+ for (size_t j=0; j<response->NumFrameActual; j++)
+ {
+ qf = p_sys->allocatedFrames[i + j];
+ if (qf->pic)
+ picture_Release(qf->pic);
+ else if (qf->surface.Data.MemId)
+ {
+ ID3D11Buffer *buffer = qf->surface.Data.MemId;
+ ID3D11Buffer_Release(buffer);
+ }
+ free(qf);
+ p_sys->allocatedFrames[i + j] = NULL;
+ }
+ break;
+ }
+ }
+ free(mids);
+ return MFX_ERR_NONE;
+}
+
+static mfxStatus qsv_frame_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
+{
+ encoder_t *enc = pthis;
+ encoder_sys_t *p_sys = enc->p_sys;
+ d3d11_decoder_device_t *d3d11_dev = GetD3D11OpaqueDevice(p_sys->dec_device);
+ QSVFrame *qf = memIdQsv(mid);
+ D3D11_MAPPED_SUBRESOURCE mappedResource;
+ HRESULT hr;
+ hr = ID3D11DeviceContext_Map(d3d11_dev->d3d_dev.d3dcontext, qf->surface.Data.MemId, 0, D3D11_MAP_READ, D3D11_MAP_FLAG_DO_NOT_WAIT, &mappedResource);
+ if (FAILED(hr))
+ return MFX_ERR_LOCK_MEMORY;
+ if (likely(!qf->pic))
+ {
+ /* P8 buffer */
+ ptr->Pitch = (mfxU16)mappedResource.RowPitch;
+ ptr->Y = (mfxU8*)mappedResource.pData;
+ ptr->U = 0;
+ ptr->V = 0;
+ }
+ return MFX_ERR_NONE;
+}
+
+static mfxStatus qsv_frame_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
+{
+ VLC_UNUSED(ptr);
+ encoder_t *enc = pthis;
+ encoder_sys_t *p_sys = enc->p_sys;
+ d3d11_decoder_device_t *d3d11_dev = GetD3D11OpaqueDevice(p_sys->dec_device);
+ QSVFrame *qf = memIdQsv(mid);
+ ID3D11DeviceContext_Unmap(d3d11_dev->d3d_dev.d3dcontext, qf->surface.Data.MemId, 0);
+ return MFX_ERR_NONE;
+}
+
+static int assert_D3D11ShadowPool( encoder_t *enc, D3D11_TEXTURE2D_DESC *desc )
+{
+ encoder_sys_t *sys = enc->p_sys;
+
+ if (sys->copy_pool[0])
+ return VLC_SUCCESS;
+
+ unsigned pool_size = desc->ArraySize;
+
+ video_format_t surface_fmt = enc->fmt_in.video;
+ /* worst case scenario we need 128 alignment for HEVC */
+ // surface_fmt.i_width = (surface_fmt.i_width + 0x7F) & ~0x7F;
+ // surface_fmt.i_height = (surface_fmt.i_height + 0x7F) & ~0x7F;
+
+ const d3d_format_t *textureFmt = GetTextureFormat(desc->Format, enc->fmt_in.video.i_chroma);
+ if (unlikely(textureFmt==NULL))
+ return VLC_EGENERIC;
+
+ d3d11_decoder_device_t *d3d11_dev = GetD3D11OpaqueDevice(sys->dec_device);
+
+ unsigned i;
+ for (i=0; i<pool_size; i++)
+ if (AllocateTextures(enc, &d3d11_dev->d3d_dev, textureFmt,
+ &surface_fmt, 1, &sys->copy_pool[i], NULL) != VLC_SUCCESS)
+ {
+ while (i-- != 0)
+ ID3D11Texture2D_Release(sys->copy_pool[i]);
+ return VLC_EGENERIC;
+ }
+ for (; i<ARRAY_SIZE(sys->copy_pool); i++)
+ sys->copy_pool[i] = NULL;
+
+ return VLC_SUCCESS;
+}
+
+static mfxStatus qsv_frame_get_hdl(mfxHDL pthis, mfxMemId mid, mfxHDL *hdl)
+{
+ encoder_t *enc = pthis;
+ if (NULL == hdl)
+ return MFX_ERR_INVALID_HANDLE;
+ QSVFrame *qf = memIdQsv(mid);
+ mfxHDLPair *pPair = (mfxHDLPair*)hdl;
+ if (!qf->pic)
+ pPair->first = qf->surface.Data.MemId;
+ else
+ {
+ picture_sys_d3d11_t *p_sys = ActiveD3D11PictureSys(qf->pic);
+ if (qf->mem_type & MFX_MEMTYPE_INTERNAL_FRAME)
+ pPair->first = p_sys->texture[0];
+ else
+ {
+ encoder_sys_t *sys = enc->p_sys;
+ D3D11_TEXTURE2D_DESC texDesc;
+ ID3D11Texture2D_GetDesc(p_sys->texture[0], &texDesc);
+ assert_D3D11ShadowPool(enc, &texDesc );
+ d3d11_decoder_device_t *d3d11_dev = GetD3D11OpaqueDevice(sys->dec_device);
+ ID3D11DeviceContext_CopySubresourceRegion(d3d11_dev->d3d_dev.d3dcontext,
+ (ID3D11Resource*)sys->copy_pool[p_sys->slice_index],
+ 0, 0, 0, 0,
+ p_sys->resource[0],
+ p_sys->slice_index,
+ NULL);
+ pPair->first = sys->copy_pool[p_sys->slice_index];
+ }
+ }
+ pPair->second = 0;
+ return MFX_ERR_NONE;
+}
+#endif // HAVE_AVCODEC_D3D11VA
+
static int Open(vlc_object_t *this)
{
encoder_t *enc = (encoder_t *)this;
@@ -405,7 +647,7 @@ static int Open(vlc_object_t *this)
#endif
};
mfxVersion ver = { { 1, 1 } };
- mfxIMPL impl;
+ mfxIMPL impl = MFX_IMPL_HARDWARE_ANY;
mfxVideoParam param_out = { 0 };
uint8_t *p_extra;
@@ -430,9 +672,11 @@ static int Open(vlc_object_t *this)
sys->dec_device = vlc_video_context_HoldDevice(enc->vctx_in);
else
sys->dec_device = vlc_encoder_GetDecoderDevice(enc);
+ if ( sys->dec_device && GetD3D11OpaqueDevice(sys->dec_device) != NULL )
+ impl = MFX_IMPL_HARDWARE_ANY | MFX_IMPL_VIA_D3D11;
/* Initialize dispatcher, it will loads the actual SW/HW Implementation */
- sts = MFXInit(MFX_IMPL_AUTO_ANY, &ver, &sys->session);
+ sts = MFXInit(impl, &ver, &sys->session);
if (sts != MFX_ERR_NONE) {
if (sts == MFX_ERR_UNSUPPORTED)
@@ -462,6 +706,28 @@ static int Open(vlc_object_t *this)
/* Input picture format description */
sys->params.mfx.FrameInfo.FrameRateExtN = enc->fmt_in.video.i_frame_rate;
sys->params.mfx.FrameInfo.FrameRateExtD = enc->fmt_in.video.i_frame_rate_base;
+
+ if ((impl & MFX_IMPL_VIA_D3D11) && GetD3D11OpaqueDevice(sys->dec_device))
+ {
+ d3d11_decoder_device_t *d3d11_dev = GetD3D11OpaqueDevice(sys->dec_device);
+ ID3D11Device *d3device;
+ ID3D11DeviceContext_GetDevice(d3d11_dev->d3d_dev.d3dcontext, &d3device);
+
+ ID3D10Multithread *pMultithread;
+ HRESULT hr = ID3D11Device_QueryInterface( d3device, &IID_ID3D10Multithread, (void **)&pMultithread);
+ if (SUCCEEDED(hr)) {
+ ID3D10Multithread_SetMultithreadProtected(pMultithread, TRUE);
+ ID3D10Multithread_Release(pMultithread);
+ }
+
+ mfxHDL handle = d3device;
+ sts = MFXVideoCORE_SetHandle(sys->session, MFX_HANDLE_D3D11_DEVICE, handle);
+ ID3D11Device_Release(d3device);
+ if (sts != MFX_ERR_NONE) {
+ msg_Err(enc, "No direct3d11 handle available.");
+ goto error;
+ }
+ }
sys->params.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12;
sys->params.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420;
sys->params.mfx.FrameInfo.Width = vlc_align(enc->fmt_in.video.i_width, 16);
@@ -476,6 +742,9 @@ static int Open(vlc_object_t *this)
/* Parsing options common to all RC methods and codecs */
sys->params.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
+ if (impl & MFX_IMPL_VIA_D3D11)
+ sys->params.IOPattern |= MFX_IOPATTERN_IN_VIDEO_MEMORY;
+ else
sys->params.IOPattern |= MFX_IOPATTERN_IN_SYSTEM_MEMORY;
sys->params.AsyncDepth = var_InheritInteger(enc, SOUT_CFG_PREFIX "async-depth");
sys->params.mfx.GopOptFlag = 1; /* TODO */
@@ -557,6 +826,26 @@ static int Open(vlc_object_t *this)
goto error;
}
+#if HAVE_AVCODEC_D3D11VA
+ if (impl & MFX_IMPL_VIA_D3D11)
+ {
+ mfxFrameAllocator frame_allocator = {
+ .pthis = enc,
+ .GetHDL = qsv_frame_get_hdl,
+ .Alloc = qsv_frame_alloc,
+ .Free = qsv_frame_free,
+ .Lock = qsv_frame_lock,
+ .Unlock = qsv_frame_unlock,
+ };
+
+ sts = MFXVideoCORE_SetFrameAllocator(sys->session, &frame_allocator);
+ if (sts != MFX_ERR_NONE)
+ {
+ msg_Err(enc, "Failed to set D3D11 frame allocator");
+ }
+ }
+#endif
+
/* Request number of surface needed and creating frame pool */
if (MFXVideoENCODE_QueryIOSurf(sys->session, &sys->params, &alloc_request)!= MFX_ERR_NONE)
{
@@ -627,8 +916,18 @@ static int Open(vlc_object_t *this)
async_task_t_fifo_Init(&sys->packets);
/* Vlc module configuration */
+ if (impl & MFX_IMPL_VIA_D3D11)
+ {
+ enc->fmt_in.i_codec = VLC_CODEC_D3D11_OPAQUE;
+ enc->fmt_in.video.i_chroma = VLC_CODEC_D3D11_OPAQUE;
+ enc->fmt_in.video.i_bits_per_pixel = 12;
+ }
+ else
+ {
enc->fmt_in.i_codec = VLC_CODEC_NV12; // Intel Media SDK requirement
enc->fmt_in.video.i_bits_per_pixel = 12;
+ enc->fmt_in.video.i_chroma = VLC_CODEC_NV12;
+ }
enc->fmt_in.video.i_width = sys->params.mfx.FrameInfo.Width;
enc->fmt_in.video.i_height = sys->params.mfx.FrameInfo.Height;
@@ -653,6 +952,14 @@ static void Close(vlc_object_t *this)
MFXVideoENCODE_Close(sys->session);
MFXClose(sys->session);
+
+#if HAVE_AVCODEC_D3D11VA
+ for (unsigned i=0; i<ARRAY_SIZE(sys->copy_pool); i++)
+ {
+ if ( sys->copy_pool[i] )
+ ID3D11Texture2D_Release( sys->copy_pool[i] );
+ }
+#endif
/* if (enc->fmt_out.p_extra) */
/* free(enc->fmt_out.p_extra); */
async_task_t_fifo_Release(&sys->packets);
@@ -749,16 +1056,21 @@ static int submit_frame(encoder_t *enc, picture_t *pic, QSVFrame **new_frame)
return ret;
}
- qf->pic = picture_pool_Get( sys->input_pool );
- if (unlikely(!qf->pic))
+ if (pic->context != NULL)
+ qf->pic = picture_Hold( pic );
+ else
{
- msg_Warn(enc, "Unable to find an unlocked surface in the pool");
- qf->used = 0;
- return ret;
- }
+ qf->pic = picture_pool_Get( sys->input_pool );
+ if (unlikely(!qf->pic))
+ {
+ msg_Warn(enc, "Unable to find an unlocked surface in the pool");
+ qf->used = 0;
+ return ret;
+ }
picture_Copy( qf->pic, pic );
assert(qf->pic->p[0].p_pixels + (qf->pic->p[0].i_pitch * qf->pic->p[0].i_lines) == qf->pic->p[1].p_pixels);
+ }
qf->surface.Info = sys->params.mfx.FrameInfo;
@@ -772,6 +1084,7 @@ static int submit_frame(encoder_t *enc, picture_t *pic, QSVFrame **new_frame)
//qf->surface.Data.Pitch = vlc_align(qf->surface.Info.Width, 16);
+ qf->surface.Data.MemId = qf;
qf->surface.Data.PitchLow = qf->pic->p[0].i_pitch;
qf->surface.Data.Y = qf->pic->p[0].p_pixels;
qf->surface.Data.UV = qf->pic->p[1].p_pixels;
--
2.26.2
More information about the vlc-devel
mailing list