[Libav-user] Race condition during executing x264_encoder_reconfig with multithreading

[TuTu Yu]

Hi, 
I found out a bug in x264 when running ffmpeg with x264 module.
When ffmpeg tried to execute x264_encoder_reconfig during run time, x264 will first initialize values in structure x264_param_t back to default.
And later, x264 will properly set variable in structure x264_param_t based on user configuration in function x264_validate_parameters. 
However, if this happens during running multithreading, un-expected encoding process will be produced. 
Here is my patch towards this change. 
Please let me know if you need me to explain more on this issue. 
Thanks
TuTu 

--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
diff --git a/encoder/encoder.c b/encoder/encoder.c
index 78ad56a..e7b0238 100644
--- a/encoder/encoder.c
+++ b/encoder/encoder.c
@@ -416,6 +416,8 @@ static void x264_encoder_thread_init( x264_t *h )
 
 static int x264_validate_parameters( x264_t *h, int b_open )
 {
+    unsigned int analyse_inter = h->param.analyse.inter;
+    unsigned int analyse_intra = h->param.analyse.intra;
 #if HAVE_MMX
     if( b_open )
     {
@@ -1003,16 +1005,18 @@ static int x264_validate_parameters( x264_t *h, int b_open )
         (h->mb.b_lossless || h->param.analyse.i_subpel_refine <= 1) )
         h->param.analyse.i_me_method = X264_ME_ESA;
     h->param.analyse.b_mixed_references = h->param.analyse.b_mixed_references && h->param.i_frame_reference > 1;
-    h->param.analyse.inter &= X264_ANALYSE_PSUB16x16|X264_ANALYSE_PSUB8x8|X264_ANALYSE_BSUB16x16|
+    analyse_inter &= X264_ANALYSE_PSUB16x16|X264_ANALYSE_PSUB8x8|X264_ANALYSE_BSUB16x16|
                               X264_ANALYSE_I4x4|X264_ANALYSE_I8x8;
-    h->param.analyse.intra &= X264_ANALYSE_I4x4|X264_ANALYSE_I8x8;
-    if( !(h->param.analyse.inter & X264_ANALYSE_PSUB16x16) )
-        h->param.analyse.inter &= ~X264_ANALYSE_PSUB8x8;
+    analyse_intra &= X264_ANALYSE_I4x4|X264_ANALYSE_I8x8;
+    if( !(analyse_inter & X264_ANALYSE_PSUB16x16) )
+        analyse_intra &= ~X264_ANALYSE_PSUB8x8;
     if( !h->param.analyse.b_transform_8x8 )
     {
-        h->param.analyse.inter &= ~X264_ANALYSE_I8x8;
-        h->param.analyse.intra &= ~X264_ANALYSE_I8x8;
+        analyse_inter &= ~X264_ANALYSE_I8x8;
+        analyse_intra &= ~X264_ANALYSE_I8x8;
     }
+    h->param.analyse.intra = analyse_intra;
+    h->param.analyse.inter = analyse_inter;
     h->param.analyse.i_trellis = x264_clip3( h->param.analyse.i_trellis, 0, 2 );
     h->param.rc.i_aq_mode = x264_clip3( h->param.rc.i_aq_mode, 0, 2 );
     h->param.rc.f_aq_strength = x264_clip3f( h->param.rc.f_aq_strength, 0, 3 );
@@ -1689,8 +1693,6 @@ int x264_encoder_reconfig( x264_t *h, x264_param_t *param )
     COPY( i_deblocking_filter_alphac0 );
     COPY( i_deblocking_filter_beta );
     COPY( i_frame_packing );
-    COPY( analyse.inter );
-    COPY( analyse.intra );
     COPY( analyse.i_direct_mv_pred );
     /* Scratch buffer prevents me_range from being increased for esa/tesa */
     if( h->param.analyse.i_me_method < X264_ME_ESA || param->analyse.i_me_range < h->param.analyse.i_me_range )