[x265] [PATCH] Fix bug in reading 32x32 scalinglists

[Mateusz Brzostek]

For command
x265 --vbv-maxrate 10000 --vbv-bufsize 10000 720p50_parkrun_ter.y4m w.hevc
in function Encoder::create() in line #278
double start = quantF / (m_scalingList.m_quantCoef[trSize][numList][QP_MAX_SPEC % 6][i]);
trSize is 0, 1, 2 and 3
numList is 1
so with trSize == 3 && numList == 1 there is crash -- in this patch there is only memory allocation for [3][0] and [3][3] indexes but not for [3][1].


W dniu 2016-09-07 o 08:55, vignesh at multicorewareinc.com pisze:
> # HG changeset patch
> # User vignesh
> # Date 1472642767 -19800
> #      Wed Aug 31 16:56:07 2016 +0530
> # Node ID e61403d77a36ad1da3d9ea50f68c12c9a6d2026a
> # Parent  e6ab763efa9509e6212b9018da7889a20676a69c
> Fix bug in reading 32x32 scalinglists
>
> diff -r e6ab763efa95 -r e61403d77a36 source/common/scalinglist.cpp
> --- a/source/common/scalinglist.cpp	Wed Jun 22 12:02:06 2016 +0530
> +++ b/source/common/scalinglist.cpp	Wed Aug 31 16:56:07 2016 +0530
> @@ -141,7 +141,7 @@
>      bool ok = true;
>      for (int sizeId = 0; sizeId < NUM_SIZES; sizeId++)
>      {
> -        for (int listId = 0; listId < NUM_LISTS; listId++)
> +        for (int listId = 0; listId < NUM_LISTS; listId += (sizeId == 3) ? 3 : 1)
>          {
>              m_scalingListCoef[sizeId][listId] = X265_MALLOC(int32_t, X265_MIN(MAX_MATRIX_COEF_NUM, s_numCoefPerSize[sizeId]));
>              ok &= !!m_scalingListCoef[sizeId][listId];
> @@ -160,7 +160,7 @@
>  {
>      for (int sizeId = 0; sizeId < NUM_SIZES; sizeId++)
>      {
> -        for (int listId = 0; listId < NUM_LISTS; listId++)
> +        for (int listId = 0; listId < NUM_LISTS; listId += (sizeId == 3) ? 3 : 1)
>          {
>              X265_FREE(m_scalingListCoef[sizeId][listId]);
>              for (int rem = 0; rem < NUM_REM; rem++)
> @@ -198,7 +198,7 @@
>      int defaultCounter = 0;
>  
>      for (int s = 0; s < NUM_SIZES; s++)
> -        for (int l = 0; l < NUM_LISTS; l++)
> +        for (int l = 0; l < NUM_LISTS; l += (s == 3) ? 3 : 1)
>              if (!memcmp(m_scalingListCoef[s][l], getScalingListDefaultAddress(s, l),
>                          sizeof(int32_t) * X265_MIN(MAX_MATRIX_COEF_NUM, s_numCoefPerSize[s])) &&
>                  ((s < BLOCK_16x16) || (m_scalingListDC[s][l] == 16)))
> @@ -237,7 +237,7 @@
>  void ScalingList::setDefaultScalingList()
>  {
>      for (int sizeId = 0; sizeId < NUM_SIZES; sizeId++)
> -        for (int listId = 0; listId < NUM_LISTS; listId++)
> +        for (int listId = 0; listId < NUM_LISTS; listId += (sizeId == 3) ? 3 : 1)
>              processDefaultMarix(sizeId, listId);
>      m_bEnabled = true;
>      m_bDataPresent = false;
> @@ -332,7 +332,7 @@
>          int stride = X265_MIN(MAX_MATRIX_SIZE_NUM, width);
>          int count = s_numCoefPerSize[size];
>  
> -        for (int list = 0; list < NUM_LISTS; list++)
> +        for (int list = 0; list < NUM_LISTS; list += (size == 3) ? 3 : 1)
>          {
>              int32_t *coeff = m_scalingListCoef[size][list];
>              int32_t dc = m_scalingListDC[size][list];
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel