[x265] [PATCH] main12: fix intra_pred_planar8x8 overflow

[chen]

Could you try to analyze the Dynamic Range, I worry about the intermedia result more than 16 bits.




At 2015-09-28 13:09:33,dnyaneshwar at multicorewareinc.com wrote:
># HG changeset patch
># User Dnyaneshwar G <dnyaneshwar at multicorewareinc.com>
># Date 1443416892 -19800
>#      Mon Sep 28 10:38:12 2015 +0530
># Node ID 21ab6be50abfb116256093a4d7e2bdb2783bc93d
># Parent  4f524673358748837d281abf28e4382f2ee845d2
>main12: fix intra_pred_planar8x8 overflow
>
>diff -r 4f5246733587 -r 21ab6be50abf source/common/x86/asm-primitives.cpp
>--- a/source/common/x86/asm-primitives.cpp	Thu Sep 24 18:53:28 2015 -0500
>+++ b/source/common/x86/asm-primitives.cpp	Mon Sep 28 10:38:12 2015 +0530
>@@ -960,8 +960,12 @@
>         ALL_LUMA_TU_S(calcresidual, getResidual, sse2);
>         ALL_LUMA_TU_S(transpose, transpose, sse2);
> 
>+        p.cu[BLOCK_4x4].intra_pred[PLANAR_IDX] = PFX(intra_pred_planar4_sse2);
>+        p.cu[BLOCK_8x8].intra_pred[PLANAR_IDX] = PFX(intra_pred_planar8_sse2);
>+
> #if X265_DEPTH <= 10
>-        ALL_LUMA_TU_S(intra_pred[PLANAR_IDX], intra_pred_planar, sse2);
>+        p.cu[BLOCK_16x16].intra_pred[PLANAR_IDX] = PFX(intra_pred_planar16_sse2);
>+        p.cu[BLOCK_32x32].intra_pred[PLANAR_IDX] = PFX(intra_pred_planar32_sse2);
> #endif /* X265_DEPTH <= 10 */
>         ALL_LUMA_TU_S(intra_pred[DC_IDX], intra_pred_dc, sse2);
> 
>@@ -1137,8 +1141,12 @@
>         p.cu[BLOCK_16x16].intra_filter = PFX(intra_filter_16x16_sse4);
>         p.cu[BLOCK_32x32].intra_filter = PFX(intra_filter_32x32_sse4);
> 
>+        p.cu[BLOCK_4x4].intra_pred[PLANAR_IDX] = PFX(intra_pred_planar4_sse4);
>+        p.cu[BLOCK_8x8].intra_pred[PLANAR_IDX] = PFX(intra_pred_planar8_sse4);
>+
> #if X265_DEPTH <= 10
>-        ALL_LUMA_TU_S(intra_pred[PLANAR_IDX], intra_pred_planar, sse4);
>+        p.cu[BLOCK_16x16].intra_pred[PLANAR_IDX] = PFX(intra_pred_planar16_sse4);
>+        p.cu[BLOCK_32x32].intra_pred[PLANAR_IDX] = PFX(intra_pred_planar32_sse4);
> #endif
>         ALL_LUMA_TU_S(intra_pred[DC_IDX], intra_pred_dc, sse4);
>         INTRA_ANG_SSE4_COMMON(sse4);
>diff -r 4f5246733587 -r 21ab6be50abf source/common/x86/intrapred16.asm
>--- a/source/common/x86/intrapred16.asm	Thu Sep 24 18:53:28 2015 -0500
>+++ b/source/common/x86/intrapred16.asm	Mon Sep 28 10:38:12 2015 +0530
>@@ -709,7 +709,7 @@
> %endif
>     pmullw          m1, [pw_planar16_mul + mmsize]
>     paddw           m1, m3
>-    psraw           m1, 4
>+    psrlw           m1, 4
>     movu            [r0], m1
> %if (%1 < 7)
>     paddw           m3, m4
>@@ -2197,7 +2197,7 @@
>     pmullw          m1, m0
>     paddw           m1, m3
>     paddw           m3, m4
>-    psraw           m1, 4
>+    psrlw           m1, 4
>     movu            [r0], m1
>     lea             r0, [r0 + r1]
> %endmacro
>_______________________________________________
>x265-devel mailing list
>x265-devel at videolan.org
>https://mailman.videolan.org/listinfo/x265-devel
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20150928/ad0c36a3/attachment-0001.html>