<div dir="ltr">From d61dd735c43d122d8d4f86ebf249971ab1391b40 Mon Sep 17 00:00:00 2001<br>From: Niranjan Kumar <<a href="mailto:niranjan@multicorewareinc.com" target="_blank">niranjan@multicorewareinc.com</a>><br>Date: Thu, 15 Sep 2022 16:05:22 +0530<br>Subject: [PATCH] Add Segment based Ratecontrol(SBRC) feature<br><br>---<br> doc/reST/cli.rst             |  5 +++<br> source/CMakeLists.txt        |  2 +-<br> source/common/common.h       |  4 ++<br> source/common/frame.cpp      |  3 +-<br> source/common/frame.h        |  4 ++<br> source/common/lowres.cpp     |  4 ++<br> source/common/lowres.h       |  3 ++<br> source/common/param.cpp      |  8 +++-<br> source/encoder/slicetype.cpp | 87 ++++++++++++++++++++++++++++++++----<br> source/x265.h                |  4 ++<br> source/x265cli.cpp           |  1 +<br> source/x265cli.h             |  2 +<br> 12 files changed, 115 insertions(+), 12 deletions(-)<br><br>diff --git a/doc/reST/cli.rst b/doc/reST/cli.rst<br>index 1640c1f8e..be32fc6cc 100755<br>--- a/doc/reST/cli.rst<br>+++ b/doc/reST/cli.rst<br>@@ -1768,6 +1768,11 @@ Quality, rate control and rate distortion options<br>       Default 1.0.<br>         **Range of values:** 0.0 to 3.0<br> <br>+.. option:: --sbrc --no-sbrc<br>+<br>+        To enable and disable segment based rate control.<br>+    Default: disabled.<br>+<br> .. option:: --hevc-aq<br> <br>   Enable adaptive quantization<br>diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt<br>index 13e4750de..8a93a8bc4 100755<br>--- a/source/CMakeLists.txt<br>+++ b/source/CMakeLists.txt<br>@@ -29,7 +29,7 @@ option(NATIVE_BUILD "Target the build CPU" OFF)<br> option(STATIC_LINK_CRT "Statically link C runtime for release builds" OFF)<br> mark_as_advanced(FPROFILE_USE FPROFILE_GENERATE NATIVE_BUILD)<br> # X265_BUILD must be incremented each time the public API is changed<br>-set(X265_BUILD 204)<br>+set(X265_BUILD 205)<br> configure_file("${PROJECT_SOURCE_DIR}/<a href="http://x265.def.in" target="_blank">x265.def.in</a>"<br>                "${PROJECT_BINARY_DIR}/x265.def")<br> configure_file("${PROJECT_SOURCE_DIR}/<a href="http://x265_config.h.in" target="_blank">x265_config.h.in</a>"<br>diff --git a/source/common/common.h b/source/common/common.h<br>index a245c7dae..9a578590a 100644<br>--- a/source/common/common.h<br>+++ b/source/common/common.h<br>@@ -131,6 +131,7 @@ typedef uint64_t pixel4;<br> typedef int64_t  ssum2_t;<br> #define SHIFT_TO_BITPLANE 9<br> #define HISTOGRAM_BINS 1024<br>+#define BRIGHTNESS_THRESHOLD 120 // The threshold above which a pixel is bright<br> #else<br> typedef uint8_t  pixel;<br> typedef uint16_t sum_t;<br>@@ -139,6 +140,7 @@ typedef uint32_t pixel4;<br> typedef int32_t  ssum2_t; // Signed sum<br> #define SHIFT_TO_BITPLANE 7<br> #define HISTOGRAM_BINS 256<br>+#define BRIGHTNESS_THRESHOLD 30 // The threshold above which a pixel is bright<br> #endif // if HIGH_BIT_DEPTH<br> <br> #if X265_DEPTH < 10<br>@@ -162,6 +164,8 @@ typedef uint64_t sse_t;<br> <br> #define MIN_QPSCALE     0.21249999999999999<br> #define MAX_MAX_QPSCALE 615.46574234477100<br>+#define FRAME_BRIGHTNESS_THRESHOLD  50.0 // Min % of pixels in a frame, that are above BRIGHTNESS_THRESHOLD for it to be considered a bright frame<br>+#define FRAME_EDGE_THRESHOLD  10.0 // Min % of edge pixels in a frame, for it to be considered to have high edge density<br> <br> <br> template<typename T><br>diff --git a/source/common/frame.cpp b/source/common/frame.cpp<br>index 255882a9d..d8b982df2 100644<br>--- a/source/common/frame.cpp<br>+++ b/source/common/frame.cpp<br>@@ -63,6 +63,7 @@ Frame::Frame()<br>     m_thetaPic = NULL;<br>     m_edgeBitPlane = NULL;<br>     m_edgeBitPic = NULL;<br>+    m_frameSegment = X265_AQ_NONE;<br>     m_isInsideWindow = 0;<br> }<br> <br>@@ -104,7 +105,7 @@ bool Frame::create(x265_param *param, float* quantOffsets)<br>         CHECKED_MALLOC_ZERO(m_classifyCount, uint32_t, size);<br>     }<br> <br>-    if (param->rc.aqMode == X265_AQ_EDGE || (param->rc.zonefileCount && param->rc.aqMode != 0))<br>+    if (param->rc.aqMode == X265_AQ_EDGE || param->rc.frameSegment || (param->rc.zonefileCount && param->rc.aqMode != 0))<br>     {<br>         uint32_t numCuInWidth = (param->sourceWidth + param->maxCUSize - 1) / param->maxCUSize;<br>         uint32_t numCuInHeight = (param->sourceHeight + param->maxCUSize - 1) / param->maxCUSize;<br>diff --git a/source/common/frame.h b/source/common/frame.h<br>index ac1185e81..485b0883d 100644<br>--- a/source/common/frame.h<br>+++ b/source/common/frame.h<br>@@ -142,6 +142,10 @@ public:<br>     pixel*                 m_edgeBitPlane;<br>     pixel*                 m_edgeBitPic;<br> <br>+    /* segment for each frame */<br>+    int                    m_frameSegment;<br>+<br>+<br>     int                    m_isInsideWindow;<br> <br>     Frame();<br>diff --git a/source/common/lowres.cpp b/source/common/lowres.cpp<br>index 578981d64..5e7718e7e 100644<br>--- a/source/common/lowres.cpp<br>+++ b/source/common/lowres.cpp<br>@@ -190,6 +190,9 @@ bool Lowres::create(x265_param* param, PicYuv *origPic, uint32_t qgSize)<br>         }<br>     }<br> <br>+    if (param->rc.frameSegment)<br>+        lowresEdgePlane = X265_MALLOC(pixel, lumaStride * (lines + (origPic->m_lumaMarginY * 2)));<br>+<br>     return true;<br> <br> fail:<br>@@ -235,6 +238,7 @@ void Lowres::destroy()<br>     X265_FREE(edgeInclined);<br>     X265_FREE(qpAqMotionOffset);<br>     X265_FREE(blockVariance);<br>+    X265_FREE(lowresEdgePlane);<br>     if (maxAQDepth > 0)<br>     {<br>         for (uint32_t d = 0; d < 4; d++)<br>diff --git a/source/common/lowres.h b/source/common/lowres.h<br>index 2a7497258..03d713edb 100644<br>--- a/source/common/lowres.h<br>+++ b/source/common/lowres.h<br>@@ -44,6 +44,9 @@ struct ReferencePlanes<br>     pixel*   fpelLowerResPlane[3];<br>     pixel*   lowerResPlane[4];<br> <br>+    /* Edge Plane in Lowres */<br>+    pixel*   lowresEdgePlane;<br>+<br>     bool     isWeighted;<br>     bool     isLowres;<br>     bool     isHMELowres;<br>diff --git a/source/common/param.cpp b/source/common/param.cpp<br>index ed6973d9c..fa16a364c 100755<br>--- a/source/common/param.cpp<br>+++ b/source/common/param.cpp<br>@@ -298,6 +298,7 @@ void x265_param_default(x265_param* param)<br>     param->rc.bEnableConstVbv = 0;<br>     param->bResetZoneConfig = 1;<br>     param->reconfigWindowSize = 0;<br>+    param->rc.frameSegment = 0;<br>     param->decoderVbvMaxRate = 0;<br>     param->bliveVBV2pass = 0;<br> <br>@@ -1250,6 +1251,7 @@ int x265_param_parse(x265_param* p, const char* name, const char* value)<br>         OPT("multi-pass-opt-analysis") p->analysisMultiPassRefine = atobool(value);<br>         OPT("multi-pass-opt-distortion") p->analysisMultiPassDistortion = atobool(value);<br>         OPT("aq-motion") p->bAQMotion = atobool(value);<br>+        OPT("sbrc") p->rc.frameSegment = atobool(value);<br>         OPT("dynamic-rd") p->dynamicRd = atof(value);<br>         OPT("analysis-reuse-level")<br>         {<br>@@ -2007,9 +2009,11 @@ void x265_print_params(x265_param* param)<br>              param->maxNumReferences, (param->limitReferences & X265_REF_LIMIT_CU) ? "on" : "off",<br>              (param->limitReferences & X265_REF_LIMIT_DEPTH) ? "on" : "off");<br> <br>-    if (param->rc.aqMode)<br>+    if (param->rc.aqMode && !param->rc.frameSegment)<br>         x265_log(param, X265_LOG_INFO, "AQ: mode / str / qg-size / cu-tree  : %d / %0.1f / %d / %d\n", param->rc.aqMode,<br>                  param->rc.aqStrength, param->rc.qgSize, param->rc.cuTree);<br>+     else if (param->rc.frameSegment)<br>+        x265_log(param, X265_LOG_INFO, "AQ: mode / str / qg-size / cu-tree  : auto / %0.1f / %d / %d\n", param->rc.aqStrength, param->rc.qgSize, param->rc.cuTree);<br> <br>     if (param->bLossless)<br>         x265_log(param, X265_LOG_INFO, "Rate Control                        : Lossless\n");<br>@@ -2323,6 +2327,7 @@ char *x265_param2string(x265_param* p, int padx, int pady)<br>     s += sprintf(s, " hist-threshold=%.2f", p->edgeTransitionThreshold);<br>     BOOL(p->bOptCUDeltaQP, "opt-cu-delta-qp");<br>     BOOL(p->bAQMotion, "aq-motion");<br>+    BOOL(p->rc.frameSegment, "sbrc");<br>     BOOL(p->bEmitHDR10SEI, "hdr10");<br>     BOOL(p->bHDR10Opt, "hdr10-opt");<br>     BOOL(p->bDhdr10opt, "dhdr10-opt");<br>@@ -2617,6 +2622,7 @@ void x265_copy_params(x265_param* dst, x265_param* src)<br>     dst->rc.bEnableConstVbv = src->rc.bEnableConstVbv;<br>     dst->rc.hevcAq = src->rc.hevcAq;<br>     dst->rc.qpAdaptationRange = src->rc.qpAdaptationRange;<br>+    dst->rc.frameSegment = src->rc.frameSegment;<br> <br>     dst->vui.aspectRatioIdc = src->vui.aspectRatioIdc;<br>     dst->vui.sarWidth = src->vui.sarWidth;<br>diff --git a/source/encoder/slicetype.cpp b/source/encoder/slicetype.cpp<br>index 1222a8511..5de1ad956 100644<br>--- a/source/encoder/slicetype.cpp<br>+++ b/source/encoder/slicetype.cpp<br>@@ -473,9 +473,9 @@ void LookaheadTLD::calcAdaptiveQuantFrame(Frame *curFrame, x265_param* param)<br>     if (!(param->rc.bStatRead && param->rc.cuTree && IS_REFERENCED(curFrame)))<br>     {<br>         /* Calculate Qp offset for each 16x16 or 8x8 block in the frame */<br>-        if (param->rc.aqMode == X265_AQ_NONE || param->rc.aqStrength == 0)<br>+        if (curFrame->m_frameSegment == X265_AQ_NONE || param->rc.aqStrength == 0)<br>         {<br>-            if (param->rc.aqMode && param->rc.aqStrength == 0)<br>+            if (curFrame->m_frameSegment && param->rc.aqStrength == 0)<br>             {<br>                 if (quantOffsets)<br>                 {<br>@@ -516,17 +516,17 @@ void LookaheadTLD::calcAdaptiveQuantFrame(Frame *curFrame, x265_param* param)<br>                 double bias_strength = 0.f;<br>                 double strength = 0.f;<br> <br>-                if (param->rc.aqMode == X265_AQ_EDGE)<br>+                if (curFrame->m_frameSegment == X265_AQ_EDGE )<br>                     edgeFilter(curFrame, param);<br> <br>-                if (param->rc.aqMode == X265_AQ_EDGE && !param->bHistBasedSceneCut && param->recursionSkipMode == EDGE_BASED_RSKIP)<br>+                if (curFrame->m_frameSegment == X265_AQ_EDGE  && !param->bHistBasedSceneCut && param->recursionSkipMode == EDGE_BASED_RSKIP)<br>                 {<br>                     pixel* src = curFrame->m_edgePic + curFrame->m_fencPic->m_lumaMarginY * curFrame->m_fencPic->m_stride + curFrame->m_fencPic->m_lumaMarginX;<br>                     primitives.planecopy_pp_shr(src, curFrame->m_fencPic->m_stride, curFrame->m_edgeBitPic,<br>                         curFrame->m_fencPic->m_stride, curFrame->m_fencPic->m_picWidth, curFrame->m_fencPic->m_picHeight, SHIFT_TO_BITPLANE);<br>                 }<br> <br>-                if (param->rc.aqMode == X265_AQ_AUTO_VARIANCE || param->rc.aqMode == X265_AQ_AUTO_VARIANCE_BIASED || param->rc.aqMode == X265_AQ_EDGE)<br>+                if (curFrame->m_frameSegment == X265_AQ_AUTO_VARIANCE || curFrame->m_frameSegment == X265_AQ_AUTO_VARIANCE_BIASED || curFrame->m_frameSegment == X265_AQ_EDGE)<br>                 {<br>                     double bit_depth_correction = 1.f / (1 << (2 * (X265_DEPTH - 8)));<br>                     for (int blockY = 0; blockY < maxRow; blockY += loopIncr)<br>@@ -535,7 +535,7 @@ void LookaheadTLD::calcAdaptiveQuantFrame(Frame *curFrame, x265_param* param)<br>                         {<br>                             uint32_t energy, edgeDensity, avgAngle;<br>                             energy = acEnergyCu(curFrame, blockX, blockY, param->internalCsp, param->rc.qgSize);<br>-                            if (param->rc.aqMode == X265_AQ_EDGE)<br>+                            if (curFrame->m_frameSegment == X265_AQ_EDGE)<br>                             {<br>                                 edgeDensity = edgeDensityCu(curFrame, avgAngle, blockX, blockY, param->rc.qgSize);<br>                                 if (edgeDensity)<br>@@ -575,17 +575,17 @@ void LookaheadTLD::calcAdaptiveQuantFrame(Frame *curFrame, x265_param* param)<br>                 {<br>                     for (int blockX = 0; blockX < maxCol; blockX += loopIncr)<br>                     {<br>-                        if (param->rc.aqMode == X265_AQ_AUTO_VARIANCE_BIASED)<br>+                        if (curFrame->m_frameSegment == X265_AQ_AUTO_VARIANCE_BIASED)<br>                         {<br>                             qp_adj = curFrame->m_lowres.qpCuTreeOffset[blockXY];<br>                             qp_adj = strength * (qp_adj - avg_adj) + bias_strength * (1.f - modeTwoConst / (qp_adj * qp_adj));<br>                         }<br>-                        else if (param->rc.aqMode == X265_AQ_AUTO_VARIANCE)<br>+                        else if (curFrame->m_frameSegment == X265_AQ_AUTO_VARIANCE)<br>                         {<br>                             qp_adj = curFrame->m_lowres.qpCuTreeOffset[blockXY];<br>                             qp_adj = strength * (qp_adj - avg_adj);<br>                         }<br>-                        else if (param->rc.aqMode == X265_AQ_EDGE)<br>+                        else if (curFrame->m_frameSegment == X265_AQ_EDGE)<br>                         {<br>                             inclinedEdge = curFrame->m_lowres.edgeInclined[blockXY];<br>                             qp_adj = curFrame->m_lowres.qpCuTreeOffset[blockXY];<br>@@ -1377,6 +1377,45 @@ void Lookahead::getEstimatedPictureCost(Frame *curFrame)<br>     }<br> }<br> <br>+double computeBrightnessIntensity(pixel *inPlane, int width, int height, intptr_t stride)<br>+{<br>+    pixel* rowStart = inPlane;<br>+    double count = 0;<br>+<br>+    for (int i = 0; i < height; i++)<br>+    {<br>+        for (int j = 0; j < width; j++)<br>+        {<br>+            if (rowStart[j] > BRIGHTNESS_THRESHOLD)<br>+                count++;<br>+        }<br>+        rowStart += stride;<br>+    }<br>+<br>+    /* Returns the brightness percentage of the input plane */<br>+    return (count / (width * height)) * 100;<br>+}<br>+<br>+double computeEdgeIntensity(pixel *inPlane, int width, int height, intptr_t stride)<br>+{<br>+    pixel* rowStart = inPlane;<br>+    double count = 0;<br>+<br>+    for (int i = 0; i < height; i++)<br>+    {<br>+        for (int j = 0; j < width; j++)<br>+        {<br>+            if (rowStart[j] > 0)<br>+                count++;<br>+        }<br>+        rowStart += stride;<br>+    }<br>+<br>+    /* Returns the edge percentage of the input plane */<br>+    return (count / (width * height)) * 100;<br>+}<br>+<br>+<br> void PreLookaheadGroup::processTasks(int workerThreadID)<br> {<br>     if (workerThreadID < 0)<br>@@ -1391,6 +1430,36 @@ void PreLookaheadGroup::processTasks(int workerThreadID)<br>         ProfileScopeEvent(prelookahead);<br>         m_lock.release();<br>         preFrame->m_lowres.init(preFrame->m_fencPic, preFrame->m_poc);<br>+<br>+        /* SBRC */<br>+        if (preFrame->m_param->rc.frameSegment)<br>+        {<br>+            int heightL = preFrame->m_lowres.lines;<br>+            int widthL = preFrame->m_lowres.width;<br>+            pixel *lumaPlane = preFrame->m_lowres.fpelPlane[0];<br>+            intptr_t stride = preFrame->m_lowres.lumaStride;<br>+            double brightnessIntensity = 0, edgeIntensity = 0;<br>+<br>+            /* Edge plane computation */<br>+            memset(preFrame->m_lowres.lowresEdgePlane, 0, stride * (heightL + (preFrame->m_fencPic->m_lumaMarginY * 2)) * sizeof(pixel));<br>+            pixel* lowresEdgePic = preFrame->m_lowres.lowresEdgePlane + preFrame->m_fencPic->m_lumaMarginY * stride + preFrame->m_fencPic->m_lumaMarginX;<br>+            computeEdge(lowresEdgePic, lumaPlane, NULL, stride, heightL, widthL, false);<br>+<br>+            /*Frame edge percentage computation */<br>+            edgeIntensity = computeEdgeIntensity(lowresEdgePic, widthL, heightL, stride);<br>+<br>+            /* Frame Brightness percentage computation */<br>+            brightnessIntensity = computeBrightnessIntensity(lumaPlane, widthL, heightL, stride);<br>+<br>+            /* AQ mode switch */<br>+            if (edgeIntensity < FRAME_EDGE_THRESHOLD)<br>+                preFrame->m_frameSegment = brightnessIntensity > FRAME_BRIGHTNESS_THRESHOLD? X265_AQ_AUTO_VARIANCE : X265_AQ_AUTO_VARIANCE_BIASED;<br>+            else<br>+                preFrame->m_frameSegment = brightnessIntensity > FRAME_BRIGHTNESS_THRESHOLD? X265_AQ_EDGE : X265_AQ_EDGE_BIASED;<br>+        }<br>+        else<br>+            preFrame->m_frameSegment = preFrame->m_param->rc.aqMode;<br>+<br>         if (m_lookahead.m_bAdaptiveQuant)<br>             tld.calcAdaptiveQuantFrame(preFrame, m_lookahead.m_param);<br>         tld.lowresIntraEstimate(preFrame->m_lowres, m_lookahead.m_param->rc.qgSize);<br>diff --git a/source/x265.h b/source/x265.h<br>index 5d242d653..787aa12fa 100644<br>--- a/source/x265.h<br>+++ b/source/x265.h<br>@@ -581,6 +581,7 @@ typedef enum<br> #define X265_AQ_AUTO_VARIANCE        2<br> #define X265_AQ_AUTO_VARIANCE_BIASED 3<br> #define X265_AQ_EDGE                 4<br>+#define X265_AQ_EDGE_BIASED          1<br> #define x265_ADAPT_RD_STRENGTH   4<br> #define X265_REFINE_INTER_LEVELS 3<br> /* NOTE! For this release only X265_CSP_I420 and X265_CSP_I444 are supported */<br>@@ -1496,6 +1497,9 @@ typedef struct x265_param<br>         /* internally enable if tune grain is set */<br>         int      bEnableConstVbv;<br> <br>+        /* enable SBRC mode for each sequence */<br>+        int      frameSegment;<br>+<br>         /* if only the focused frames would be re-encode or not */<br>         int       bEncFocusedFramesOnly;<br> <br>diff --git a/source/x265cli.cpp b/source/x265cli.cpp<br>index f6ba0ab30..c004b7915 100755<br>--- a/source/x265cli.cpp<br>+++ b/source/x265cli.cpp<br>@@ -262,6 +262,7 @@ namespace X265_NS {<br>         H0("   --aq-strength <float>         Reduces blocking and blurring in flat and textured areas (0 to 3.0). Default %.2f\n", param->rc.aqStrength);<br>         H0("   --qp-adaptation-range <float> Delta QP range by QP adaptation based on a psycho-visual model (1.0 to 6.0). Default %.2f\n", param->rc.qpAdaptationRange);<br>         H0("   --[no-]aq-motion              Block level QP adaptation based on the relative motion between the block and the frame. Default %s\n", OPT(param->bAQMotion));<br>+        H1("   --[no-]sbrc                   Enables the segment based rate control, using its scene statistics. Default %s\n", OPT(param->rc.frameSegment));<br>         H0("   --qg-size <int>               Specifies the size of the quantization group (64, 32, 16, 8). Default %d\n", param->rc.qgSize);<br>         H0("   --[no-]cutree                 Enable cutree for Adaptive Quantization. Default %s\n", OPT(param->rc.cuTree));<br>         H0("   --[no-]rc-grain               Enable ratecontrol mode to handle grains specifically. turned on with tune grain. Default %s\n", OPT(param->rc.bEnableGrain));<br>diff --git a/source/x265cli.h b/source/x265cli.h<br>index 7072f7616..8640a5bbb 100644<br>--- a/source/x265cli.h<br>+++ b/source/x265cli.h<br>@@ -184,6 +184,8 @@ static const struct option long_options[] =<br>     { "qp",             required_argument, NULL, 'q' },<br>     { "aq-mode",        required_argument, NULL, 0 },<br>     { "aq-strength",    required_argument, NULL, 0 },<br>+    { "sbrc",                 no_argument, NULL, 0 },<br>+    { "no-sbrc",              no_argument, NULL, 0 },<br>     { "rc-grain",             no_argument, NULL, 0 },<br>     { "no-rc-grain",          no_argument, NULL, 0 },<br>     { "ipratio",        required_argument, NULL, 0 },<br>-- <br>2.37.2.windows.2<br><br><div><br></div>-- <br><div dir="ltr" data-smartmail="gmail_signature"><div dir="ltr"><div><div dir="ltr"><div dir="ltr"><div dir="ltr"><font color="#0c343d" face="verdana, sans-serif"><br></font></div><div dir="ltr"><font color="#0c343d" face="verdana, sans-serif">Thanks & Regards</font><div><font color="#0c343d" face="verdana, sans-serif"><b>Niranjan Kumar B</b></font></div><div><font size="1" color="#0c343d" face="verdana, sans-serif">Video Codec Engineer </font></div><div><font size="1" color="#0c343d" face="verdana, sans-serif">Media & AI Analytics</font></div><div><font face="trebuchet ms, sans-serif" color="#0c343d">+91 958 511 1449</font></div><div><a href="https://multicorewareinc.com/" style="color:rgb(17,85,204)" target="_blank"><img src="https://docs.google.com/uc?export=download&id=1kc3RJu9M8bnIf6Xa5rUw2d-eEVUsPBE5&revid=0B7tw9XJBmynaemR1VUpQUi9DVytRVW5SVkRwVTFjb1hBMUcwPQ"></a></div></div></div></div></div></div></div></div>