[x264-devel] Fix possible non-determinism with mbtree + open-gop + sync-lookahead

[Anton Mitrofanov]

x264 | branch: master | Anton Mitrofanov <BugMaster at narod.ru> | Thu Jan 24 12:11:26 2013 +0400| [c13fbaf279d41e6bb8db09e95aec1b638ff026e8] | committer: Jason Garrett-Glaser

Fix possible non-determinism with mbtree + open-gop + sync-lookahead

Code assumed keyframe analysis would only pull one frame off the list; this
isn't true with open-gop.

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=c13fbaf279d41e6bb8db09e95aec1b638ff026e8
---

 encoder/analyse.h   |    2 +-
 encoder/lookahead.c |   10 ++++++----
 encoder/slicetype.c |    9 +++++++--
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/encoder/analyse.h b/encoder/analyse.h
index 8aa318d..cbbb0df 100644
--- a/encoder/analyse.h
+++ b/encoder/analyse.h
@@ -34,7 +34,7 @@ void x264_analyse_weight_frame( x264_t *h, int end );
 void x264_macroblock_analyse( x264_t *h );
 void x264_slicetype_decide( x264_t *h );
 
-void x264_slicetype_analyse( x264_t *h, int keyframe );
+void x264_slicetype_analyse( x264_t *h, int intra_minigop );
 
 int x264_weighted_reference_duplicate( x264_t *h, int i_ref, const x264_weight_t *w );
 
diff --git a/encoder/lookahead.c b/encoder/lookahead.c
index c99f117..ad8a3e6 100644
--- a/encoder/lookahead.c
+++ b/encoder/lookahead.c
@@ -70,18 +70,19 @@ static void x264_lookahead_slicetype_decide( x264_t *h )
     x264_stack_align( x264_slicetype_decide, h );
 
     x264_lookahead_update_last_nonb( h, h->lookahead->next.list[0] );
+    int shift_frames = h->lookahead->next.list[0]->i_bframes + 1;
 
     x264_pthread_mutex_lock( &h->lookahead->ofbuf.mutex );
     while( h->lookahead->ofbuf.i_size == h->lookahead->ofbuf.i_max_size )
         x264_pthread_cond_wait( &h->lookahead->ofbuf.cv_empty, &h->lookahead->ofbuf.mutex );
 
     x264_pthread_mutex_lock( &h->lookahead->next.mutex );
-    x264_lookahead_shift( &h->lookahead->ofbuf, &h->lookahead->next, h->lookahead->next.list[0]->i_bframes + 1 );
+    x264_lookahead_shift( &h->lookahead->ofbuf, &h->lookahead->next, shift_frames );
     x264_pthread_mutex_unlock( &h->lookahead->next.mutex );
 
     /* For MB-tree and VBV lookahead, we have to perform propagation analysis on I-frames too. */
     if( h->lookahead->b_analyse_keyframe && IS_X264_TYPE_I( h->lookahead->last_nonb->i_type ) )
-        x264_stack_align( x264_slicetype_analyse, h, 1 );
+        x264_stack_align( x264_slicetype_analyse, h, shift_frames );
 
     x264_pthread_mutex_unlock( &h->lookahead->ofbuf.mutex );
 }
@@ -236,11 +237,12 @@ void x264_lookahead_get_frames( x264_t *h )
 
         x264_stack_align( x264_slicetype_decide, h );
         x264_lookahead_update_last_nonb( h, h->lookahead->next.list[0] );
-        x264_lookahead_shift( &h->lookahead->ofbuf, &h->lookahead->next, h->lookahead->next.list[0]->i_bframes + 1 );
+        int shift_frames = h->lookahead->next.list[0]->i_bframes + 1;
+        x264_lookahead_shift( &h->lookahead->ofbuf, &h->lookahead->next, shift_frames );
 
         /* For MB-tree and VBV lookahead, we have to perform propagation analysis on I-frames too. */
         if( h->lookahead->b_analyse_keyframe && IS_X264_TYPE_I( h->lookahead->last_nonb->i_type ) )
-            x264_stack_align( x264_slicetype_analyse, h, 1 );
+            x264_stack_align( x264_slicetype_analyse, h, shift_frames );
 
         x264_lookahead_encoder_shift( h );
     }
diff --git a/encoder/slicetype.c b/encoder/slicetype.c
index 06fdb12..5d77776 100644
--- a/encoder/slicetype.c
+++ b/encoder/slicetype.c
@@ -1393,7 +1393,7 @@ static int scenecut( x264_t *h, x264_mb_analysis_t *a, x264_frame_t **frames, in
     return scenecut_internal( h, a, frames, p0, p1, real_scenecut );
 }
 
-void x264_slicetype_analyse( x264_t *h, int keyframe )
+void x264_slicetype_analyse( x264_t *h, int intra_minigop )
 {
     x264_mb_analysis_t a;
     x264_frame_t *frames[X264_LOOKAHEAD_MAX+3] = { NULL, };
@@ -1402,8 +1402,13 @@ void x264_slicetype_analyse( x264_t *h, int keyframe )
     int cost1p0, cost2p0, cost1b1, cost2p1;
     int i_max_search = X264_MIN( h->lookahead->next.i_size, X264_LOOKAHEAD_MAX );
     int vbv_lookahead = h->param.rc.i_vbv_buffer_size && h->param.rc.i_lookahead;
+    /* For determinism we should limit the search to the number of frames lookahead has for sure
+     * in h->lookahead->next.list buffer, except at the end of stream.
+     * For normal calls with (intra_minigop == 0) that is h->lookahead->i_slicetype_length + 1 frames.
+     * And for I-frame calls (intra_minigop != 0) we already removed intra_minigop frames from there. */
     if( h->param.b_deterministic )
-        i_max_search = X264_MIN( i_max_search, h->lookahead->i_slicetype_length + !keyframe );
+        i_max_search = X264_MIN( i_max_search, h->lookahead->i_slicetype_length + 1 - intra_minigop );
+    int keyframe = !!intra_minigop;
 
     assert( h->frames.b_have_lowres );