[x265] [PATCH] Fix seg fault while enabling sea

[Karam Singh]

This patch has been pushed to the master branch.
*__________________________*
*Karam Singh*
*Ph.D. IIT Guwahati*
Senior Software (Video Coding) Engineer
Mobile: +91 8011279030
Block 9A, 6th floor, DLF Cyber City
Manapakkam, Chennai 600 089


On Fri, Sep 27, 2024 at 3:32 PM Anusuya Kumarasamy <
anusuya.kumarasamy at multicorewareinc.com> wrote:

> From 7cacb15e7992677942cacf858c80698a2af8d5b3 Mon Sep 17 00:00:00 2001
> From: AnusuyaKumarasamy <anusuya.kumarasamy at multicorewareinc.com>
> Date: Wed, 18 Sep 2024 16:29:29 +0530
> Subject: [PATCH 1/5] Fix seg fault while enabling sea
>
> ---
>  source/encoder/encoder.cpp     | 28 ++++++++++++++++------------
>  source/encoder/framefilter.cpp |  6 +++---
>  source/encoder/framefilter.h   |  2 +-
>  3 files changed, 20 insertions(+), 16 deletions(-)
>
> diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp
> index 65c247aba..82123606c 100644
> --- a/source/encoder/encoder.cpp
> +++ b/source/encoder/encoder.cpp
> @@ -2332,22 +2332,26 @@ int Encoder::encode(const x265_picture* pic_in,
> x265_picture** pic_out)
>                      }
>                  }
>              }
> -            if (m_param->searchMethod == X265_SEA &&
> frameEnc[0]->m_lowres.sliceType != X265_TYPE_B)
> +
> +            for (int layer = 0; layer < m_param->numLayers; layer++)
>              {
> -                int padX = m_param->maxCUSize + 32;
> -                int padY = m_param->maxCUSize + 16;
> -                uint32_t numCuInHeight =
> (frameEnc[0]->m_encData->m_reconPic[0]->m_picHeight + m_param->maxCUSize -
> 1) / m_param->maxCUSize;
> -                int maxHeight = numCuInHeight * m_param->maxCUSize;
> -                for (int i = 0; i < INTEGRAL_PLANE_NUM; i++)
> +                if (m_param->searchMethod == X265_SEA &&
> (frameEnc[layer]->m_lowres.sliceType != X265_TYPE_B || !layer))
>                  {
> -                    frameEnc[0]->m_encData->m_meBuffer[i] =
> X265_MALLOC(uint32_t, frameEnc[0]->m_reconPic[0]->m_stride * (maxHeight +
> (2 * padY)));
> -                    if (frameEnc[0]->m_encData->m_meBuffer[i])
> +                    int padX = m_param->maxCUSize + 32;
> +                    int padY = m_param->maxCUSize + 16;
> +                    uint32_t numCuInHeight =
> (frameEnc[layer]->m_encData->m_reconPic[0]->m_picHeight +
> m_param->maxCUSize - 1) / m_param->maxCUSize;
> +                    int maxHeight = numCuInHeight * m_param->maxCUSize;
> +                    for (int i = 0; i < INTEGRAL_PLANE_NUM; i++)
>                      {
> -                        memset(frameEnc[0]->m_encData->m_meBuffer[i], 0,
> sizeof(uint32_t)* frameEnc[0]->m_reconPic[0]->m_stride * (maxHeight + (2 *
> padY)));
> -                        frameEnc[0]->m_encData->m_meIntegral[i] =
> frameEnc[0]->m_encData->m_meBuffer[i] +
> frameEnc[0]->m_encData->m_reconPic[0]->m_stride * padY + padX;
> +                        frameEnc[layer]->m_encData->m_meBuffer[i] =
> X265_MALLOC(uint32_t, frameEnc[layer]->m_reconPic[0]->m_stride * (maxHeight
> + (2 * padY)));
> +                        if (frameEnc[layer]->m_encData->m_meBuffer[i])
> +                        {
> +
>  memset(frameEnc[layer]->m_encData->m_meBuffer[i], 0, sizeof(uint32_t) *
> frameEnc[layer]->m_reconPic[0]->m_stride * (maxHeight + (2 * padY)));
> +                            frameEnc[layer]->m_encData->m_meIntegral[i] =
> frameEnc[layer]->m_encData->m_meBuffer[i] +
> frameEnc[layer]->m_encData->m_reconPic[0]->m_stride * padY + padX;
> +                        }
> +                        else
> +                            x265_log(m_param, X265_LOG_ERROR, "SEA motion
> search: POC %d Integral buffer[%d] unallocated\n", frameEnc[0]->m_poc, i);
>                      }
> -                    else
> -                        x265_log(m_param, X265_LOG_ERROR, "SEA motion
> search: POC %d Integral buffer[%d] unallocated\n", frameEnc[0]->m_poc, i);
>                  }
>              }
>
> diff --git a/source/encoder/framefilter.cpp
> b/source/encoder/framefilter.cpp
> index 344ac738d..71dc42b2c 100644
> --- a/source/encoder/framefilter.cpp
> +++ b/source/encoder/framefilter.cpp
> @@ -659,7 +659,7 @@ void FrameFilter::processPostRow(int row, int layer)
>
>      /* Generate integral planes for SEA motion search */
>      if(m_param->searchMethod == X265_SEA)
> -        computeMEIntegral(row);
> +        computeMEIntegral(row, layer);
>      // Notify other FrameEncoders that this row of reconstructed pixels
> is available
>      m_frame->m_reconRowFlag[row].set(1);
>
> @@ -722,10 +722,10 @@ void FrameFilter::processPostRow(int row, int layer)
>      }
>  }
>
> -void FrameFilter::computeMEIntegral(int row)
> +void FrameFilter::computeMEIntegral(int row, int layer)
>  {
>      int lastRow = row ==
> (int)m_frame->m_encData->m_slice->m_sps->numCuInHeight - 1;
> -    if (m_frame->m_lowres.sliceType != X265_TYPE_B)
> +    if (m_frame->m_lowres.sliceType != X265_TYPE_B || !layer)
>      {
>          /* If WPP, other than first row, integral calculation for current
> row needs to wait till the
>          * integral for the previous row is computed */
> diff --git a/source/encoder/framefilter.h b/source/encoder/framefilter.h
> index b84b7094c..8bea1a0e9 100644
> --- a/source/encoder/framefilter.h
> +++ b/source/encoder/framefilter.h
> @@ -130,7 +130,7 @@ public:
>
>      void processRow(int row, int layer);
>      void processPostRow(int row, int layer);
> -    void computeMEIntegral(int row);
> +    void computeMEIntegral(int row, int layer);
>  };
>  }
>
> --
> 2.36.0.windows.1
>
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20241004/d249174d/attachment.htm>