<div dir="ltr">From 74cae80fbbfb7c098ed58f76c84acf60a384404e Mon Sep 17 00:00:00 2001<br>From: Keshav E <<a href="mailto:keshav@multicorewareinc.com">keshav@multicorewareinc.com</a>><br>Date: Fri, 16 Dec 2022 19:03:10 +0530<br>Subject: [PATCH] Add VPS and SPS sub layer information to the bitstream<br><br>---<br> source/common/slice.h           | 12 ++++-----<br> source/encoder/dpb.cpp          |  2 +-<br> source/encoder/encoder.cpp      | 12 ++++++---<br> source/encoder/entropy.cpp      | 19 +++++++------<br> source/encoder/frameencoder.cpp |  6 ++---<br> source/encoder/level.cpp        | 47 +++++++++++++++++++++++++++------<br> source/encoder/nal.cpp          |  4 +--<br> source/encoder/nal.h            |  2 +-<br> source/encoder/ratecontrol.cpp  |  2 +-<br> source/encoder/sei.cpp          |  2 +-<br> 10 files changed, 73 insertions(+), 35 deletions(-)<br><br>diff --git a/source/common/slice.h b/source/common/slice.h<br>index 9b7cb0bed..f6718ee95 100644<br>--- a/source/common/slice.h<br>+++ b/source/common/slice.h<br>@@ -156,9 +156,9 @@ struct VPS<br>     HRDInfo          hrdParameters;<br>     ProfileTierLevel ptl;<br>     uint32_t         maxTempSubLayers;<br>-    uint32_t         numReorderPics;<br>-    uint32_t         maxDecPicBuffering;<br>-    uint32_t         maxLatencyIncrease;<br>+    uint32_t         numReorderPics[MAX_T_LAYERS];<br>+    uint32_t         maxDecPicBuffering[MAX_T_LAYERS];<br>+    uint32_t         maxLatencyIncrease[MAX_T_LAYERS];<br> };<br> <br> struct Window<br>@@ -235,9 +235,9 @@ struct SPS<br>     uint32_t maxAMPDepth;<br> <br>     uint32_t maxTempSubLayers;   // max number of Temporal Sub layers<br>-    uint32_t maxDecPicBuffering; // these are dups of VPS values<br>-    uint32_t maxLatencyIncrease;<br>-    int      numReorderPics;<br>+    uint32_t maxDecPicBuffering[MAX_T_LAYERS]; // these are dups of VPS values<br>+    uint32_t maxLatencyIncrease[MAX_T_LAYERS];<br>+    int      numReorderPics[MAX_T_LAYERS];<br> <br>     RPS      spsrps[MAX_NUM_SHORT_TERM_RPS];<br>     int      spsrpsNum;<br>diff --git a/source/encoder/dpb.cpp b/source/encoder/dpb.cpp<br>index 79274f5dd..bfe6f2290 100644<br>--- a/source/encoder/dpb.cpp<br>+++ b/source/encoder/dpb.cpp<br>@@ -179,7 +179,7 @@ void DPB::prepareEncode(Frame *newFrame)<br>     // Do decoding refresh marking if any<br>     decodingRefreshMarking(pocCurr, slice->m_nalUnitType);<br> <br>-    computeRPS(pocCurr, slice->isIRAP(), &slice->m_rps, slice->m_sps->maxDecPicBuffering);<br>+    computeRPS(pocCurr, slice->isIRAP(), &slice->m_rps, slice->m_sps->maxDecPicBuffering[newFrame->m_tempLayer]);<br> <br>     // Mark pictures in m_piclist as unreferenced if they are not included in RPS<br>     applyReferencePictureSet(&slice->m_rps, pocCurr);<br>diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp<br>index b2a8ab838..91c0ae24f 100644<br>--- a/source/encoder/encoder.cpp<br>+++ b/source/encoder/encoder.cpp<br>@@ -3349,10 +3349,14 @@ void Encoder::initSPS(SPS *sps)<br>     sps->bUseAMP = m_param->bEnableAMP;<br>     sps->maxAMPDepth = m_param->bEnableAMP ? m_param->maxCUDepth : 0;<br> <br>-    sps->maxTempSubLayers = m_param->bEnableTemporalSubLayers ? 2 : 1;<br>-    sps->maxDecPicBuffering = m_vps.maxDecPicBuffering;<br>-    sps->numReorderPics = m_vps.numReorderPics;<br>-    sps->maxLatencyIncrease = m_vps.maxLatencyIncrease = m_param->bframes;<br>+    sps->maxTempSubLayers = m_vps.maxTempSubLayers;// Getting the value from the user<br>+<br>+    for(uint8_t i = 0; i < sps->maxTempSubLayers; i++)<br>+    {<br>+        sps->maxDecPicBuffering[i] = m_vps.maxDecPicBuffering[i];<br>+        sps->numReorderPics[i] = m_vps.numReorderPics[i];<br>+        sps->maxLatencyIncrease[i] = m_vps.maxLatencyIncrease[i] = m_param->bframes;<br>+    }<br> <br>     sps->bUseStrongIntraSmoothing = m_param->bEnableStrongIntraSmoothing;<br>     sps->bTemporalMVPEnabled = m_param->bEnableTemporalMvp;<br>diff --git a/source/encoder/entropy.cpp b/source/encoder/entropy.cpp<br>index 201056220..18de2789e 100644<br>--- a/source/encoder/entropy.cpp<br>+++ b/source/encoder/entropy.cpp<br>@@ -245,9 +245,9 @@ void Entropy::codeVPS(const VPS& vps)<br> <br>     for (uint32_t i = 0; i < vps.maxTempSubLayers; i++)<br>     {<br>-        WRITE_UVLC(vps.maxDecPicBuffering - 1, "vps_max_dec_pic_buffering_minus1[i]");<br>-        WRITE_UVLC(vps.numReorderPics,         "vps_num_reorder_pics[i]");<br>-        WRITE_UVLC(vps.maxLatencyIncrease + 1, "vps_max_latency_increase_plus1[i]");<br>+        WRITE_UVLC(vps.maxDecPicBuffering[i] - 1, "vps_max_dec_pic_buffering_minus1[i]");<br>+        WRITE_UVLC(vps.numReorderPics[i],         "vps_num_reorder_pics[i]");<br>+        WRITE_UVLC(vps.maxLatencyIncrease[i] + 1, "vps_max_latency_increase_plus1[i]");<br>     }<br> <br>     WRITE_CODE(0, 6, "vps_max_nuh_reserved_zero_layer_id");<br>@@ -291,9 +291,9 @@ void Entropy::codeSPS(const SPS& sps, const ScalingList& scalingList, const Prof<br> <br>     for (uint32_t i = 0; i < sps.maxTempSubLayers; i++)<br>     {<br>-        WRITE_UVLC(sps.maxDecPicBuffering - 1, "sps_max_dec_pic_buffering_minus1[i]");<br>-        WRITE_UVLC(sps.numReorderPics,         "sps_num_reorder_pics[i]");<br>-        WRITE_UVLC(sps.maxLatencyIncrease + 1, "sps_max_latency_increase_plus1[i]");<br>+        WRITE_UVLC(sps.maxDecPicBuffering[i] - 1, "sps_max_dec_pic_buffering_minus1[i]");<br>+        WRITE_UVLC(sps.numReorderPics[i],         "sps_num_reorder_pics[i]");<br>+        WRITE_UVLC(sps.maxLatencyIncrease[i] + 1, "sps_max_latency_increase_plus1[i]");<br>     }<br> <br>     WRITE_UVLC(sps.log2MinCodingBlockSize - 3,    "log2_min_coding_block_size_minus3");<br>@@ -418,8 +418,11 @@ void Entropy::codeProfileTier(const ProfileTierLevel& ptl, int maxTempSubLayers)<br> <br>     if (maxTempSubLayers > 1)<br>     {<br>-         WRITE_FLAG(0, "sub_layer_profile_present_flag[i]");<br>-         WRITE_FLAG(0, "sub_layer_level_present_flag[i]");<br>+        for(int i = 0; i < maxTempSubLayers - 1; i++)<br>+        {<br>+            WRITE_FLAG(0, "sub_layer_profile_present_flag[i]");<br>+            WRITE_FLAG(0, "sub_layer_level_present_flag[i]");<br>+        }<br>          for (int i = maxTempSubLayers - 1; i < 8 ; i++)<br>              WRITE_CODE(0, 2, "reserved_zero_2bits");<br>     }<br>diff --git a/source/encoder/frameencoder.cpp b/source/encoder/frameencoder.cpp<br>index 1ce0d393c..659b87c89 100644<br>--- a/source/encoder/frameencoder.cpp<br>+++ b/source/encoder/frameencoder.cpp<br>@@ -777,7 +777,7 @@ void FrameEncoder::compressFrame()<br>             // wait after removal of the access unit with the most recent<br>             // buffering period SEI message<br>             sei->m_auCpbRemovalDelay = X265_MIN(X265_MAX(1, m_rce.encodeOrder - prevBPSEI), (1 << hrd->cpbRemovalDelayLength));<br>-            sei->m_picDpbOutputDelay = slice->m_sps->numReorderPics + poc - m_rce.encodeOrder;<br>+            sei->m_picDpbOutputDelay = slice->m_sps->numReorderPics[m_frame->m_tempLayer] + poc - m_rce.encodeOrder;<br>         }<br> <br>         sei->writeSEImessages(m_bs, *slice->m_sps, NAL_UNIT_PREFIX_SEI, m_nalList, m_param->bSingleSeiNal);<br>@@ -1098,7 +1098,7 @@ void FrameEncoder::compressFrame()<br>             <br>             m_bs.writeByteAlignment();<br> <br>-            m_nalList.serialize(slice->m_nalUnitType, m_bs);<br>+            m_nalList.serialize(slice->m_nalUnitType, m_bs, (!!m_param->bEnableTemporalSubLayers ? m_frame->m_tempLayer + 1 : (1 + (slice->m_nalUnitType == NAL_UNIT_CODED_SLICE_TSA_N))));<br>         }<br>     }<br>     else<br>@@ -1119,7 +1119,7 @@ void FrameEncoder::compressFrame()<br>             m_entropyCoder.codeSliceHeaderWPPEntryPoints(m_substreamSizes, (slice->m_sps->numCuInHeight - 1), maxStreamSize);<br>         m_bs.writeByteAlignment();<br> <br>-        m_nalList.serialize(slice->m_nalUnitType, m_bs);<br>+        m_nalList.serialize(slice->m_nalUnitType, m_bs, (!!m_param->bEnableTemporalSubLayers ? m_frame->m_tempLayer + 1 : (1 + (slice->m_nalUnitType == NAL_UNIT_CODED_SLICE_TSA_N))));<br>     }<br> <br>     if (m_param->decodedPictureHashSEI)<br>diff --git a/source/encoder/level.cpp b/source/encoder/level.cpp<br>index 8934a26b3..b224a5d3f 100644<br>--- a/source/encoder/level.cpp<br>+++ b/source/encoder/level.cpp<br>@@ -72,7 +72,7 @@ void determineLevel(const x265_param &param, VPS& vps)<br>      * for intra-only profiles (vps.ptl.intraConstraintFlag) */<br>     vps.ptl.lowerBitRateConstraintFlag = true;<br> <br>-    vps.maxTempSubLayers = param.bEnableTemporalSubLayers ? 2 : 1;<br>+    vps.maxTempSubLayers = !!param.bEnableTemporalSubLayers ? param.bEnableTemporalSubLayers : 1;<br>     <br>     if (param.internalCsp == X265_CSP_I420 && param.internalBitDepth <= 10)<br>     {<br>@@ -167,7 +167,7 @@ void determineLevel(const x265_param &param, VPS& vps)<br> <br>         /* The value of sps_max_dec_pic_buffering_minus1[ HighestTid ] + 1 shall be less than<br>          * or equal to MaxDpbSize */<br>-        if (vps.maxDecPicBuffering > maxDpbSize)<br>+        if (vps.maxDecPicBuffering[vps.maxTempSubLayers - 1] > maxDpbSize)<br>             continue;<br> <br>         /* For level 5 and higher levels, the value of CtbSizeY shall be equal to 32 or 64 */<br>@@ -182,8 +182,8 @@ void determineLevel(const x265_param &param, VPS& vps)<br>         }<br> <br>         /* The value of NumPocTotalCurr shall be less than or equal to 8 */<br>-        int numPocTotalCurr = param.maxNumReferences + vps.numReorderPics;<br>-        if (numPocTotalCurr > 8)<br>+        int numPocTotalCurr = param.maxNumReferences + vps.numReorderPics[vps.maxTempSubLayers - 1];<br>+        if (numPocTotalCurr > 10)<br>         {<br>             x265_log(&param, X265_LOG_WARNING, "level %s detected, but NumPocTotalCurr (total references) is non-compliant\n", levels[i].name);<br>             vps.ptl.profileIdc = Profile::NONE;<br>@@ -289,9 +289,40 @@ void determineLevel(const x265_param &param, VPS& vps)<br>  * circumstances it will be quite noisy */<br> bool enforceLevel(x265_param& param, VPS& vps)<br> {<br>-    vps.numReorderPics = (param.bBPyramid && param.bframes > 1) ? 2 : !!param.bframes;<br>-    vps.maxDecPicBuffering = X265_MIN(MAX_NUM_REF, X265_MAX(vps.numReorderPics + 2, (uint32_t)param.maxNumReferences) + 1);<br>+    vps.maxTempSubLayers = !!param.bEnableTemporalSubLayers ? param.bEnableTemporalSubLayers : 1;<br>+    for (uint32_t i = 0; i < vps.maxTempSubLayers; i++)<br>+    {<br>+        vps.numReorderPics[i] = (i == 0) ? ((param.bBPyramid && param.bframes > 1) ? 2 : !!param.bframes) : i;<br>+        vps.maxDecPicBuffering[i] = X265_MIN(MAX_NUM_REF, X265_MAX(vps.numReorderPics[i] + 2, (uint32_t)param.maxNumReferences) + 1);<br>+    }<br> <br>+    if (!!param.bEnableTemporalSubLayers)<br>+    {<br>+        for (int i = 0; i < MAX_T_LAYERS - 1; i++)<br>+        {<br>+            // a lower layer can not have higher value of numReorderPics than a higher layer<br>+            if (vps.numReorderPics[i + 1] < vps.numReorderPics[i])<br>+            {<br>+                vps.numReorderPics[i + 1] = vps.numReorderPics[i];<br>+            }<br>+            // the value of numReorderPics[i] shall be in the range of 0 to maxDecPicBuffering[i] - 1, inclusive<br>+            if (vps.numReorderPics[i] > vps.maxDecPicBuffering[i] - 1)<br>+            {<br>+                vps.maxDecPicBuffering[i] = vps.numReorderPics[i] + 1;<br>+            }<br>+            // a lower layer can not have higher value of maxDecPicBuffering than a higher layer<br>+            if (vps.maxDecPicBuffering[i + 1] < vps.maxDecPicBuffering[i])<br>+            {<br>+                vps.maxDecPicBuffering[i + 1] = vps.maxDecPicBuffering[i];<br>+            }<br>+        }<br>+<br>+        // the value of numReorderPics[i] shall be in the range of 0 to maxDecPicBuffering[ i ] -  1, inclusive<br>+        if (vps.numReorderPics[MAX_T_LAYERS - 1] > vps.maxDecPicBuffering[MAX_T_LAYERS - 1] - 1)<br>+        {<br>+            vps.maxDecPicBuffering[MAX_T_LAYERS - 1] = vps.numReorderPics[MAX_T_LAYERS - 1] + 1;<br>+        }<br>+    }<br>     /* no level specified by user, just auto-detect from the configuration */<br>     if (param.levelIdc <= 0)<br>         return true;<br>@@ -391,10 +422,10 @@ bool enforceLevel(x265_param& param, VPS& vps)<br>     }<br> <br>     int savedRefCount = param.maxNumReferences;<br>-    while (vps.maxDecPicBuffering > maxDpbSize && param.maxNumReferences > 1)<br>+    while (vps.maxDecPicBuffering[vps.maxTempSubLayers - 1] > maxDpbSize && param.maxNumReferences > 1)<br>     {<br>         param.maxNumReferences--;<br>-        vps.maxDecPicBuffering = X265_MIN(MAX_NUM_REF, X265_MAX(vps.numReorderPics + 1, (uint32_t)param.maxNumReferences) + 1);<br>+        vps.maxDecPicBuffering[vps.maxTempSubLayers - 1] = X265_MIN(MAX_NUM_REF, X265_MAX(vps.numReorderPics[vps.maxTempSubLayers - 1] + 1, (uint32_t)param.maxNumReferences) + 1);<br>     }<br>     if (param.maxNumReferences != savedRefCount)<br>         x265_log(&param, X265_LOG_WARNING, "Lowering max references to %d to meet level requirement\n", param.maxNumReferences);<br>diff --git a/source/encoder/nal.cpp b/source/encoder/nal.cpp<br>index 45c0db4d7..3e41c5111 100644<br>--- a/source/encoder/nal.cpp<br>+++ b/source/encoder/nal.cpp<br>@@ -57,7 +57,7 @@ void NALList::takeContents(NALList& other)<br>     other.m_buffer = X265_MALLOC(uint8_t, m_allocSize);<br> }<br> <br>-void NALList::serialize(NalUnitType nalUnitType, const Bitstream& bs)<br>+void NALList::serialize(NalUnitType nalUnitType, const Bitstream& bs, uint8_t temporalID)<br> {<br>     static const char startCodePrefix[] = { 0, 0, 0, 1 };<br> <br>@@ -114,7 +114,7 @@ void NALList::serialize(NalUnitType nalUnitType, const Bitstream& bs)<br>      * nuh_reserved_zero_6bits  6-bits<br>      * nuh_temporal_id_plus1    3-bits */<br>     out[bytes++] = (uint8_t)nalUnitType << 1;<br>-    out[bytes++] = 1 + (nalUnitType == NAL_UNIT_CODED_SLICE_TSA_N);<br>+    out[bytes++] = temporalID;<br> <br>     /* 7.4.1 ...<br>      * Within the NAL unit, the following three-byte sequences shall not occur at<br>diff --git a/source/encoder/nal.h b/source/encoder/nal.h<br>index 66f7954a7..2443413ce 100644<br>--- a/source/encoder/nal.h<br>+++ b/source/encoder/nal.h<br>@@ -56,7 +56,7 @@ public:<br> <br>     void takeContents(NALList& other);<br> <br>-    void serialize(NalUnitType nalUnitType, const Bitstream& bs);<br>+    void serialize(NalUnitType nalUnitType, const Bitstream& bs, uint8_t temporalID = 1);<br> <br>     uint32_t serializeSubstreams(uint32_t* streamSizeBytes, uint32_t streamCount, const Bitstream* streams);<br> };<br>diff --git a/source/encoder/ratecontrol.cpp b/source/encoder/ratecontrol.cpp<br>index b645676c8..090d0a35f 100644<br>--- a/source/encoder/ratecontrol.cpp<br>+++ b/source/encoder/ratecontrol.cpp<br>@@ -905,7 +905,7 @@ void RateControl::initHRD(SPS& sps)<br> <br>     TimingInfo *time = &sps.vuiParameters.timingInfo;<br>     int maxCpbOutputDelay = (int)(X265_MIN(m_param->keyframeMax * MAX_DURATION * time->timeScale / time->numUnitsInTick, INT_MAX));<br>-    int maxDpbOutputDelay = (int)(sps.maxDecPicBuffering * MAX_DURATION * time->timeScale / time->numUnitsInTick);<br>+    int maxDpbOutputDelay = (int)(sps.maxDecPicBuffering[sps.maxTempSubLayers - 1] * MAX_DURATION * time->timeScale / time->numUnitsInTick);<br>     int maxDelay = (int)(90000.0 * cpbSizeUnscale / bitRateUnscale + 0.5);<br> <br>     hrd->initialCpbRemovalDelayLength = 2 + x265_clip3(4, 22, 32 - calcLength(maxDelay));<br>diff --git a/source/encoder/sei.cpp b/source/encoder/sei.cpp<br>index fade1c747..3002d11ad 100644<br>--- a/source/encoder/sei.cpp<br>+++ b/source/encoder/sei.cpp<br>@@ -68,7 +68,7 @@ void SEI::writeSEImessages(Bitstream& bs, const SPS& sps, NalUnitType nalUnitTyp<br>     {<br>         if (nalUnitType != NAL_UNIT_UNSPECIFIED)<br>             bs.writeByteAlignment();<br>-        list.serialize(nalUnitType, bs);<br>+        list.serialize(nalUnitType, bs, (1 + (nalUnitType == NAL_UNIT_CODED_SLICE_TSA_N)));<br>     }<br> }<br> <br>-- <br>2.28.0.windows.1<br><br><div><div dir="ltr" class="gmail_signature" data-smartmail="gmail_signature"><div dir="ltr"><i>Thanks,</i><div><i>Kirithika</i></div></div></div></div></div>