[x265] [PATCH] Fix seg fault while enabling sea

[Anusuya Kumarasamy]

>From 7cacb15e7992677942cacf858c80698a2af8d5b3 Mon Sep 17 00:00:00 2001
From: AnusuyaKumarasamy <anusuya.kumarasamy at multicorewareinc.com>
Date: Wed, 18 Sep 2024 16:29:29 +0530
Subject: [PATCH 1/5] Fix seg fault while enabling sea

---
 source/encoder/encoder.cpp     | 28 ++++++++++++++++------------
 source/encoder/framefilter.cpp |  6 +++---
 source/encoder/framefilter.h   |  2 +-
 3 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp
index 65c247aba..82123606c 100644
--- a/source/encoder/encoder.cpp
+++ b/source/encoder/encoder.cpp
@@ -2332,22 +2332,26 @@ int Encoder::encode(const x265_picture* pic_in,
x265_picture** pic_out)
                     }
                 }
             }
-            if (m_param->searchMethod == X265_SEA &&
frameEnc[0]->m_lowres.sliceType != X265_TYPE_B)
+
+            for (int layer = 0; layer < m_param->numLayers; layer++)
             {
-                int padX = m_param->maxCUSize + 32;
-                int padY = m_param->maxCUSize + 16;
-                uint32_t numCuInHeight =
(frameEnc[0]->m_encData->m_reconPic[0]->m_picHeight + m_param->maxCUSize -
1) / m_param->maxCUSize;
-                int maxHeight = numCuInHeight * m_param->maxCUSize;
-                for (int i = 0; i < INTEGRAL_PLANE_NUM; i++)
+                if (m_param->searchMethod == X265_SEA &&
(frameEnc[layer]->m_lowres.sliceType != X265_TYPE_B || !layer))
                 {
-                    frameEnc[0]->m_encData->m_meBuffer[i] =
X265_MALLOC(uint32_t, frameEnc[0]->m_reconPic[0]->m_stride * (maxHeight +
(2 * padY)));
-                    if (frameEnc[0]->m_encData->m_meBuffer[i])
+                    int padX = m_param->maxCUSize + 32;
+                    int padY = m_param->maxCUSize + 16;
+                    uint32_t numCuInHeight =
(frameEnc[layer]->m_encData->m_reconPic[0]->m_picHeight +
m_param->maxCUSize - 1) / m_param->maxCUSize;
+                    int maxHeight = numCuInHeight * m_param->maxCUSize;
+                    for (int i = 0; i < INTEGRAL_PLANE_NUM; i++)
                     {
-                        memset(frameEnc[0]->m_encData->m_meBuffer[i], 0,
sizeof(uint32_t)* frameEnc[0]->m_reconPic[0]->m_stride * (maxHeight + (2 *
padY)));
-                        frameEnc[0]->m_encData->m_meIntegral[i] =
frameEnc[0]->m_encData->m_meBuffer[i] +
frameEnc[0]->m_encData->m_reconPic[0]->m_stride * padY + padX;
+                        frameEnc[layer]->m_encData->m_meBuffer[i] =
X265_MALLOC(uint32_t, frameEnc[layer]->m_reconPic[0]->m_stride * (maxHeight
+ (2 * padY)));
+                        if (frameEnc[layer]->m_encData->m_meBuffer[i])
+                        {
+
 memset(frameEnc[layer]->m_encData->m_meBuffer[i], 0, sizeof(uint32_t) *
frameEnc[layer]->m_reconPic[0]->m_stride * (maxHeight + (2 * padY)));
+                            frameEnc[layer]->m_encData->m_meIntegral[i] =
frameEnc[layer]->m_encData->m_meBuffer[i] +
frameEnc[layer]->m_encData->m_reconPic[0]->m_stride * padY + padX;
+                        }
+                        else
+                            x265_log(m_param, X265_LOG_ERROR, "SEA motion
search: POC %d Integral buffer[%d] unallocated\n", frameEnc[0]->m_poc, i);
                     }
-                    else
-                        x265_log(m_param, X265_LOG_ERROR, "SEA motion
search: POC %d Integral buffer[%d] unallocated\n", frameEnc[0]->m_poc, i);
                 }
             }

diff --git a/source/encoder/framefilter.cpp b/source/encoder/framefilter.cpp
index 344ac738d..71dc42b2c 100644
--- a/source/encoder/framefilter.cpp
+++ b/source/encoder/framefilter.cpp
@@ -659,7 +659,7 @@ void FrameFilter::processPostRow(int row, int layer)

     /* Generate integral planes for SEA motion search */
     if(m_param->searchMethod == X265_SEA)
-        computeMEIntegral(row);
+        computeMEIntegral(row, layer);
     // Notify other FrameEncoders that this row of reconstructed pixels is
available
     m_frame->m_reconRowFlag[row].set(1);

@@ -722,10 +722,10 @@ void FrameFilter::processPostRow(int row, int layer)
     }
 }

-void FrameFilter::computeMEIntegral(int row)
+void FrameFilter::computeMEIntegral(int row, int layer)
 {
     int lastRow = row ==
(int)m_frame->m_encData->m_slice->m_sps->numCuInHeight - 1;
-    if (m_frame->m_lowres.sliceType != X265_TYPE_B)
+    if (m_frame->m_lowres.sliceType != X265_TYPE_B || !layer)
     {
         /* If WPP, other than first row, integral calculation for current
row needs to wait till the
         * integral for the previous row is computed */
diff --git a/source/encoder/framefilter.h b/source/encoder/framefilter.h
index b84b7094c..8bea1a0e9 100644
--- a/source/encoder/framefilter.h
+++ b/source/encoder/framefilter.h
@@ -130,7 +130,7 @@ public:

     void processRow(int row, int layer);
     void processPostRow(int row, int layer);
-    void computeMEIntegral(int row);
+    void computeMEIntegral(int row, int layer);
 };
 }

-- 
2.36.0.windows.1
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20240927/20cd13bd/attachment.htm>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0001-Fix-seg-fault-while-enabling-sea.patch
Type: application/octet-stream
Size: 5027 bytes
Desc: not available
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20240927/20cd13bd/attachment.obj>