[x265] [PATCH MV-HEVC 04/10] Add support for MV-HEVC in DPB and Sub DPB
Anusuya Kumarasamy
anusuya.kumarasamy at multicorewareinc.com
Tue Aug 6 10:43:06 UTC 2024
>From 19d4c3239c543b429dc05a9b80cba46eec5c8a5a Mon Sep 17 00:00:00 2001
From: Kirithika <kirithika at multicorewareinc.com>
Date: Fri, 12 Jul 2024 17:45:05 +0530
Subject: [PATCH] Add support for MV-HEVC in DPB and Sub DPB
---
source/common/frame.cpp | 34 +++++++++++++++
source/common/frame.h | 7 ++++
source/common/piclist.cpp | 86 +++++++++++++++++++++++++++++++++++++-
source/common/piclist.h | 10 +++++
source/encoder/dpb.cpp | 50 ++++++++++++++--------
source/encoder/encoder.cpp | 20 ++++++++-
6 files changed, 185 insertions(+), 22 deletions(-)
diff --git a/source/common/frame.cpp b/source/common/frame.cpp
index 79474333f..c40093dba 100644
--- a/source/common/frame.cpp
+++ b/source/common/frame.cpp
@@ -75,6 +75,11 @@ Frame::Frame()
m_tempLayer = 0;
m_sameLayerRefPic = false;
+
+ m_viewId = 0;
+ m_valid = 0;
+ m_nextSubDPB = NULL;
+ m_prevSubDPB = NULL;
}
bool Frame::create(x265_param *param, float* quantOffsets)
@@ -244,6 +249,35 @@ void Frame::destroy()
m_encData = NULL;
}
+#if ENABLE_MULTIVIEW
+ //Destroy interlayer References
+ if (refPicSetInterLayer0.size())
+ {
+ Frame* iterFrame = refPicSetInterLayer0.first();
+
+ while (iterFrame)
+ {
+ Frame* curFrame = iterFrame;
+ iterFrame = iterFrame->m_nextSubDPB;
+ refPicSetInterLayer0.removeSubDPB(*curFrame);
+ iterFrame = refPicSetInterLayer0.first();
+ }
+ }
+
+ if (refPicSetInterLayer1.size())
+ {
+ Frame* iterFrame = refPicSetInterLayer1.first();
+
+ while (iterFrame)
+ {
+ Frame* curFrame = iterFrame;
+ iterFrame = iterFrame->m_nextSubDPB;
+ refPicSetInterLayer1.removeSubDPB(*curFrame);
+ iterFrame = refPicSetInterLayer1.first();
+ }
+ }
+#endif
+
if (m_fencPic)
{
if (m_param->bCopyPicToFrame)
diff --git a/source/common/frame.h b/source/common/frame.h
index 97094ad73..4ddd8bd46 100644
--- a/source/common/frame.h
+++ b/source/common/frame.h
@@ -88,6 +88,9 @@ public:
PicYuv* m_fencPicSubsampled2;
PicYuv* m_fencPicSubsampled4;
+ PicList refPicSetInterLayer0;
+ PicList refPicSetInterLayer1;
+
int m_poc;
int m_encodeOrder;
int m_gopOffset;
@@ -164,6 +167,10 @@ public:
int m_sLayerId;
bool m_valid;
+ int m_viewId;
+ Frame* m_nextSubDPB; // PicList doubly
linked list pointers
+ Frame* m_prevSubDPB;
+
Frame();
bool create(x265_param *param, float* quantOffsets);
diff --git a/source/common/piclist.cpp b/source/common/piclist.cpp
index 345fd02c9..9c1bea7e8 100644
--- a/source/common/piclist.cpp
+++ b/source/common/piclist.cpp
@@ -82,6 +82,82 @@ void PicList::pushBack(Frame& curFrame)
m_count++;
}
+#if ENABLE_MULTIVIEW
+Frame* PicList::popFrontSubDPB()
+{
+ if (m_start)
+ {
+ Frame* temp = m_start;
+ m_count--;
+
+ if (m_count)
+ {
+ m_start = m_start->m_nextSubDPB;
+ m_start->m_prevSubDPB = NULL;
+ }
+ else
+ {
+ m_start = m_end = NULL;
+ }
+ temp->m_next = temp->m_prev = NULL;
+ return temp;
+ }
+ else
+ return NULL;
+}
+
+void PicList::pushBackSubDPB(Frame& curFrame)
+{
+ X265_CHECK(!curFrame.m_nextSubDPB && !curFrame.m_prevSubDPB, "piclist:
picture already in Sub DPB list\n"); // ensure frame is not in a list
+ curFrame.m_nextSubDPB = NULL;
+ curFrame.m_prevSubDPB = m_end;
+
+ if (m_count)
+ {
+ m_end->m_nextSubDPB = &curFrame;
+ m_end = &curFrame;
+ }
+ else
+ {
+ m_start = m_end = &curFrame;
+ }
+ m_count++;
+}
+
+void PicList::removeSubDPB(Frame& curFrame)
+{
+#if _DEBUG
+ Frame* tmp = m_start;
+ while (tmp && tmp != &curFrame)
+ {
+ tmp = tmp->m_nextSubDPB;
+ }
+
+ X265_CHECK(tmp == &curFrame, "piclist: pic being removed was not in
list\n"); // verify pic is in this list
+#endif
+
+ m_count--;
+ if (m_count)
+ {
+ if (m_start == &curFrame)
+ m_start = curFrame.m_nextSubDPB;
+ if (m_end == &curFrame)
+ m_end = curFrame.m_prevSubDPB;
+
+ if (curFrame.m_nextSubDPB)
+ curFrame.m_nextSubDPB->m_prevSubDPB = curFrame.m_prevSubDPB;
+ if (curFrame.m_prevSubDPB)
+ curFrame.m_prevSubDPB->m_nextSubDPB = curFrame.m_nextSubDPB;
+ }
+ else
+ {
+ m_start = m_end = NULL;
+ }
+
+ curFrame.m_nextSubDPB = curFrame.m_prevSubDPB = NULL;
+}
+#endif
+
void PicList::pushBackMCSTF(Frame& curFrame)
{
X265_CHECK(!curFrame.m_nextMCSTF && !curFrame.m_prevMCSTF, "piclist:
picture already in OPB list\n"); // ensure frame is not in a list
@@ -126,8 +202,13 @@ Frame *PicList::popFront()
Frame* PicList::getPOC(int poc, int sLayerId)
{
Frame *curFrame = m_start;
- while (curFrame && (curFrame->m_poc != poc || curFrame->m_sLayerId !=
sLayerId))
+ int layer = curFrame->m_param->numViews > 1 ? curFrame->m_viewId :
(curFrame->m_param->numScalableLayers > 1) ? curFrame->m_sLayerId : 0;
+ while (curFrame && (curFrame->m_poc != poc || layer != sLayerId))
+ {
curFrame = curFrame->m_next;
+ if(curFrame)
+ layer = curFrame->m_param->numViews > 1 ? curFrame->m_viewId :
(curFrame->m_param->numScalableLayers > 1) ? curFrame->m_sLayerId : 0;
+ }
return curFrame;
}
@@ -188,7 +269,8 @@ Frame *PicList::popBackMCSTF()
Frame* PicList::getCurFrame(int sLayer)
{
Frame *curFrame = m_start;
- if (curFrame->m_sLayerId == sLayer && curFrame != NULL)
+ int layer = curFrame->m_param->numViews > 1 ? curFrame->m_viewId :
(curFrame->m_param->numScalableLayers > 1) ? curFrame->m_sLayerId : 0;
+ if (layer == sLayer && curFrame != NULL)
return curFrame;
else
return NULL;
diff --git a/source/common/piclist.h b/source/common/piclist.h
index 3c392f0cb..d270d0aec 100644
--- a/source/common/piclist.h
+++ b/source/common/piclist.h
@@ -50,10 +50,16 @@ public:
/** Push picture to end of the list */
void pushBack(Frame& pic);
void pushBackMCSTF(Frame& pic);
+#if ENABLE_MULTIVIEW
+ void pushBackSubDPB(Frame& pic);
+#endif
/** Push picture to beginning of the list */
void pushFront(Frame& pic);
void pushFrontMCSTF(Frame& pic);
+#if ENABLE_MULTIVIEW
+ Frame* popFrontSubDPB();
+#endif
/** Pop picture from end of the list */
Frame* popBack();
@@ -77,6 +83,10 @@ public:
Frame* removeFrame(Frame& pic);
/* Remove MCSTF picture from list */
void removeMCSTF(Frame& pic);
+#if ENABLE_MULTIVIEW
+ /** Remove picture from Sub list */
+ void removeSubDPB(Frame& pic);
+#endif
Frame* first() { return m_start; }
diff --git a/source/encoder/dpb.cpp b/source/encoder/dpb.cpp
index 95ad41523..bce49b500 100644
--- a/source/encoder/dpb.cpp
+++ b/source/encoder/dpb.cpp
@@ -95,6 +95,10 @@ void DPB::recycleUnreferenced()
// iterator is invalidated by remove, restart scan
m_picList.remove(*curFrame);
+ if (!curFrame->m_viewId && m_picList.getPOC(curFrame->m_poc,
1) && curFrame == m_picList.getPOC(curFrame->m_poc,
1)->refPicSetInterLayer0.getPOC(curFrame->m_poc, curFrame->m_viewId))
+ {
+ m_picList.getPOC(curFrame->m_poc,
1)->refPicSetInterLayer0.removeSubDPB(*curFrame);
+ }
iterFrame = m_picList.first();
m_freeList.pushBack(*curFrame);
@@ -177,7 +181,8 @@ void DPB::prepareEncode(Frame *newFrame)
m_picList.pushFront(*newFrame);
- if (m_bTemporalSublayer &&
getTemporalLayerNonReferenceFlag(newFrame->m_sLayerId))
+ int layer = slice->m_param->numViews > 1 ? newFrame->m_viewId :
(slice->m_param->numScalableLayers > 1) ? newFrame->m_sLayerId : 0;
+ if (m_bTemporalSublayer && getTemporalLayerNonReferenceFlag(layer))
{
switch (slice->m_nalUnitType)
{
@@ -195,12 +200,12 @@ void DPB::prepareEncode(Frame *newFrame)
}
}
// Do decoding refresh marking if any
- decodingRefreshMarking(pocCurr, slice->m_nalUnitType,
newFrame->m_sLayerId);
+ decodingRefreshMarking(pocCurr, slice->m_nalUnitType, layer);
- computeRPS(pocCurr, newFrame->m_tempLayer, slice->isIRAP(),
&slice->m_rps, slice->m_sps->maxDecPicBuffering[newFrame->m_tempLayer],
newFrame->m_sLayerId);
+ computeRPS(pocCurr, newFrame->m_tempLayer, slice->isIRAP(),
&slice->m_rps, slice->m_sps->maxDecPicBuffering[newFrame->m_tempLayer],
layer);
bool isTSAPic = ((slice->m_nalUnitType == 2) || (slice->m_nalUnitType
== 3)) ? true : false;
// Mark pictures in m_piclist as unreferenced if they are not included
in RPS
- applyReferencePictureSet(&slice->m_rps, pocCurr,
newFrame->m_tempLayer, isTSAPic, newFrame->m_sLayerId);
+ applyReferencePictureSet(&slice->m_rps, pocCurr,
newFrame->m_tempLayer, isTSAPic, layer);
if (m_bTemporalSublayer && newFrame->m_tempLayer > 0
@@ -210,9 +215,9 @@ void DPB::prepareEncode(Frame *newFrame)
|| slice->m_nalUnitType == NAL_UNIT_CODED_SLICE_RASL_R)
)
{
- if (isTemporalLayerSwitchingPoint(pocCurr, newFrame->m_tempLayer,
newFrame->m_sLayerId) || (slice->m_sps->maxTempSubLayers == 1))
+ if (isTemporalLayerSwitchingPoint(pocCurr, newFrame->m_tempLayer,
layer) || (slice->m_sps->maxTempSubLayers == 1))
{
- if (getTemporalLayerNonReferenceFlag(newFrame->m_sLayerId))
+ if (getTemporalLayerNonReferenceFlag(layer))
{
slice->m_nalUnitType = NAL_UNIT_CODED_SLICE_TSA_N;
}
@@ -221,7 +226,7 @@ void DPB::prepareEncode(Frame *newFrame)
slice->m_nalUnitType = NAL_UNIT_CODED_SLICE_TSA_R;
}
}
- else if (isStepwiseTemporalLayerSwitchingPoint(&slice->m_rps,
pocCurr, newFrame->m_tempLayer, newFrame->m_sLayerId))
+ else if (isStepwiseTemporalLayerSwitchingPoint(&slice->m_rps,
pocCurr, newFrame->m_tempLayer, layer))
{
bool isSTSA = true;
int id = newFrame->m_gopOffset %
x265_gop_ra_length[newFrame->m_gopId];
@@ -254,7 +259,7 @@ void DPB::prepareEncode(Frame *newFrame)
}
if (isSTSA == true)
{
- if (getTemporalLayerNonReferenceFlag(newFrame->m_sLayerId))
+ if (getTemporalLayerNonReferenceFlag(layer))
{
slice->m_nalUnitType = NAL_UNIT_CODED_SLICE_STSA_N;
}
@@ -266,12 +271,14 @@ void DPB::prepareEncode(Frame *newFrame)
}
}
+ if (newFrame->m_viewId)
+ slice->createInterLayerReferencePictureSet(m_picList,
newFrame->refPicSetInterLayer0, newFrame->refPicSetInterLayer1);
if (slice->m_sliceType != I_SLICE)
- slice->m_numRefIdx[0] = x265_clip3(1,
newFrame->m_param->maxNumReferences, slice->m_rps.numberOfNegativePictures);
+ slice->m_numRefIdx[0] = x265_clip3(1,
newFrame->m_param->maxNumReferences, slice->m_rps.numberOfNegativePictures
+ newFrame->refPicSetInterLayer0.size() +
newFrame->refPicSetInterLayer1.size());
else
slice->m_numRefIdx[0] =
X265_MIN(newFrame->m_param->maxNumReferences,
slice->m_rps.numberOfNegativePictures); // Ensuring L0 contains just the
-ve POC
- slice->m_numRefIdx[1] = X265_MIN(newFrame->m_param->bBPyramid ? 2 : 1,
slice->m_rps.numberOfPositivePictures);
- slice->setRefPicList(m_picList, newFrame->m_sLayerId);
+ slice->m_numRefIdx[1] = X265_MIN(newFrame->m_param->bBPyramid ? 3 : 2,
slice->m_rps.numberOfPositivePictures +
newFrame->refPicSetInterLayer0.size() +
newFrame->refPicSetInterLayer1.size());
+ slice->setRefPicList(m_picList, newFrame->refPicSetInterLayer0,
newFrame->refPicSetInterLayer1, layer);
X265_CHECK(slice->m_sliceType != B_SLICE || slice->m_numRefIdx[1], "B
slice without L1 references (non-fatal)\n");
@@ -316,7 +323,8 @@ void DPB::computeRPS(int curPoc, int tempId, bool
isRAP, RPS * rps, unsigned int
while (iterPic && (poci < maxDecPicBuffer - 1))
{
- if (iterPic->m_valid && (iterPic->m_poc != curPoc) &&
iterPic->m_encData->m_bHasReferences && iterPic->m_sLayerId ==
scalableLayerId)
+ int layer = iterPic->m_param->numViews > 1 ? iterPic->m_viewId :
(iterPic->m_param->numScalableLayers > 1) ? iterPic->m_sLayerId : 0;
+ if (iterPic->m_valid && (iterPic->m_poc != curPoc) &&
iterPic->m_encData->m_bHasReferences && layer == scalableLayerId)
{
if ((!m_bTemporalSublayer || (iterPic->m_tempLayer <= tempId))
&& ((m_lastIDR >= curPoc) || (m_lastIDR <= iterPic->m_poc)))
{
@@ -340,7 +348,8 @@ void DPB::computeRPS(int curPoc, int tempId, bool
isRAP, RPS * rps, unsigned int
bool DPB::getTemporalLayerNonReferenceFlag(int scalableLayerId)
{
Frame* curFrame = m_picList.first();
- if (curFrame->m_valid && curFrame->m_encData->m_bHasReferences &&
curFrame->m_sLayerId == scalableLayerId)
+ int layer = curFrame->m_param->numViews > 1 ? curFrame->m_viewId :
(curFrame->m_param->numScalableLayers > 1) ? curFrame->m_sLayerId : 0;
+ if (curFrame->m_valid && curFrame->m_encData->m_bHasReferences &&
layer == scalableLayerId)
{
curFrame->m_sameLayerRefPic = true;
return false;
@@ -359,7 +368,8 @@ void DPB::decodingRefreshMarking(int pocCurr,
NalUnitType nalUnitType, int scala
Frame* iterFrame = m_picList.first();
while (iterFrame)
{
- if (iterFrame->m_valid && iterFrame->m_poc != pocCurr &&
iterFrame->m_sLayerId == scalableLayerId)
+ int layer = iterFrame->m_param->numViews > 1 ?
iterFrame->m_viewId : (iterFrame->m_param->numScalableLayers > 1) ?
iterFrame->m_sLayerId : 0;
+ if (iterFrame->m_valid && iterFrame->m_poc != pocCurr && layer
== scalableLayerId)
iterFrame->m_encData->m_bHasReferences = false;
iterFrame = iterFrame->m_next;
}
@@ -376,7 +386,8 @@ void DPB::decodingRefreshMarking(int pocCurr,
NalUnitType nalUnitType, int scala
Frame* iterFrame = m_picList.first();
while (iterFrame)
{
- if (iterFrame->m_valid && iterFrame->m_poc != pocCurr &&
iterFrame->m_poc != m_pocCRA && iterFrame->m_sLayerId == scalableLayerId)
+ int layer = iterFrame->m_param->numViews > 1 ?
iterFrame->m_viewId : (iterFrame->m_param->numScalableLayers > 1) ?
iterFrame->m_sLayerId : 0;
+ if (iterFrame->m_valid && iterFrame->m_poc != pocCurr &&
iterFrame->m_poc != m_pocCRA && layer == scalableLayerId)
iterFrame->m_encData->m_bHasReferences = false;
iterFrame = iterFrame->m_next;
}
@@ -404,7 +415,8 @@ void DPB::applyReferencePictureSet(RPS *rps, int
curPoc, int tempId, bool isTSAP
Frame* iterFrame = m_picList.first();
while (iterFrame)
{
- if (iterFrame->m_valid && iterFrame->m_poc != curPoc &&
iterFrame->m_encData->m_bHasReferences && iterFrame->m_sLayerId ==
scalableLayerId)
+ int layer = iterFrame->m_param->numViews > 1 ? iterFrame->m_viewId
: (iterFrame->m_param->numScalableLayers > 1) ? iterFrame->m_sLayerId : 0;
+ if (iterFrame->m_valid && iterFrame->m_poc != curPoc &&
iterFrame->m_encData->m_bHasReferences && layer == scalableLayerId)
{
// loop through all pictures in the Reference Picture Set
// to see if the picture should be kept as reference picture
@@ -447,7 +459,8 @@ bool DPB::isTemporalLayerSwitchingPoint(int curPoc, int
tempId, int scalableLaye
Frame* iterFrame = m_picList.first();
while (iterFrame)
{
- if (iterFrame->m_valid && iterFrame->m_poc != curPoc &&
iterFrame->m_encData->m_bHasReferences && iterFrame->m_sLayerId ==
scalableLayerId)
+ int layer = iterFrame->m_param->numViews > 1 ? iterFrame->m_viewId
: (iterFrame->m_param->numScalableLayers > 1) ? iterFrame->m_sLayerId : 0;
+ if (iterFrame->m_valid && iterFrame->m_poc != curPoc &&
iterFrame->m_encData->m_bHasReferences && layer == scalableLayerId)
{
if (iterFrame->m_tempLayer >= tempId)
{
@@ -465,7 +478,8 @@ bool DPB::isStepwiseTemporalLayerSwitchingPoint(RPS
*rps, int curPoc, int tempId
Frame* iterFrame = m_picList.first();
while (iterFrame)
{
- if (iterFrame->m_valid && iterFrame->m_poc != curPoc &&
iterFrame->m_encData->m_bHasReferences && iterFrame->m_sLayerId ==
scalableLayerId)
+ int layer = iterFrame->m_param->numViews > 1 ? iterFrame->m_viewId
: (iterFrame->m_param->numScalableLayers > 1) ? iterFrame->m_sLayerId : 0;
+ if (iterFrame->m_valid && iterFrame->m_poc != curPoc &&
iterFrame->m_encData->m_bHasReferences && layer == scalableLayerId)
{
for (int i = 0; i < rps->numberOfPositivePictures +
rps->numberOfNegativePictures; i++)
{
diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp
index a79ad6dab..ef152f2ae 100644
--- a/source/encoder/encoder.cpp
+++ b/source/encoder/encoder.cpp
@@ -1663,7 +1663,11 @@ int Encoder::encode(const x265_picture* pic_in,
x265_picture** pic_out)
inFrame[layer]->m_isInsideWindow = 0;
inFrame[layer]->m_tempLayer = 0;
inFrame[layer]->m_sameLayerRefPic = 0;
+#if ENABLE_MULTIVIEW
+ inFrame[layer]->m_viewId = layer;
+#else
inFrame[layer]->m_sLayerId = layer;
+#endif
inFrame[layer]->m_valid = false;
inFrame[layer]->m_lowres.bKeyframe = false;
}
@@ -1893,6 +1897,12 @@ int Encoder::encode(const x265_picture* pic_in,
x265_picture** pic_out)
if(m_param->numScalableLayers > 1)
m_dpb->m_picList.pushBack(*inFrame[1]); /* Add enhancement
layer to DPB to be used later in frameencoder*/
#endif
+
+#if ENABLE_MULTIVIEW
+ for (int view = 1; view < m_param->numViews; view++)
+ m_dpb->m_picList.pushBack(*inFrame[view]);
+#endif
+
m_numDelayedPic++;
}
else if (m_latestParam->forceFlush == 2)
@@ -2166,13 +2176,19 @@ int Encoder::encode(const x265_picture* pic_in,
x265_picture** pic_out)
frameEnc[0] = m_lookahead->getDecidedPicture();
if (frameEnc[0] && !pass && (!m_param->chunkEnd ||
(m_encodedFrameNum < m_param->chunkEnd)))
{
-#if ENABLE_ALPHA
+#if ENABLE_ALPHA || ENABLE_MULTIVIEW
//Pop non base view pictures from DPB piclist
- for (int layer = 1; layer < m_param->numScalableLayers;
layer++)
+ int numLayers = m_param->numViews > 1 ? m_param->numViews :
(m_param->numScalableLayers > 1) ? m_param->numScalableLayers : 1;
+ for (int layer = 1; layer < numLayers; layer++)
{
Frame* currentFrame =
m_dpb->m_picList.getPOC(frameEnc[0]->m_poc, layer);
frameEnc[layer] =
m_dpb->m_picList.removeFrame(*currentFrame);
+#if ENABLE_ALPHA
frameEnc[layer]->m_lowres.sliceType =
frameEnc[0]->m_lowres.sliceType;
+#else
+ int baseViewType = frameEnc[0]->m_lowres.sliceType;
+ frameEnc[layer]->m_lowres.sliceType =
IS_X265_TYPE_I(baseViewType) ? X265_TYPE_P : baseViewType;
+#endif
}
#endif
--
2.36.0.windows.1
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20240806/6a80509b/attachment-0001.htm>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0004-Add-support-for-MV-HEVC-in-DPB-and-Sub-DPB.patch
Type: application/x-patch
Size: 19553 bytes
Desc: not available
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20240806/6a80509b/attachment-0001.bin>
More information about the x265-devel
mailing list