[vlc-devel] [PATCH 1 of 6] dshow: prevent division by zero

[Edouard Gomez]

# HG changeset patch
# User Edouard Gomez <ed.gomez at free.fr>
# Date 1223648709 -7200
# Node ID ecd69aeb057e1909f51ce1d71773152f5b681e0c
# Parent  b27ef6b3315e592c09bb7cccf4320aa3158ced2d
dshow: prevent division by zero

In the DShow capture module, division by zero is possible in
multiple modulo statements when dealing with granularity values
returned by capture filters.

Change the code so that:
 - if granularity is zero, then width/height/samplingrate must be
   a perfect match with requested settings.
 - if granularity is non zero, then use the old condition tests.

This fixes real crashes reported by VLC users on different forums.

See:
http://forum.doom9.org/archive/index.php/t-134655.html
http://forum.videolan.org/viewtopic.php?f=2&t=37056&p=114065&hilit=blackmagic

Tested with both a Decklink HD Extreme card and a Decklink HD Pro card.

diff --git a/modules/access/dshow/dshow.cpp b/modules/access/dshow/dshow.cpp
--- a/modules/access/dshow/dshow.cpp
+++ b/modules/access/dshow/dshow.cpp
@@ -1318,9 +1318,14 @@
 
                                 if( i_width )
                                 {
-                                    if( i_width % pVSCC->OutputGranularityX
-                                     || pVSCC->MinOutputSize.cx > i_width
-                                     || i_width > pVSCC->MaxOutputSize.cx )
+                                    if((   !pVSCC->OutputGranularityX
+                                           && i_width != pVSCC->MinOutputSize.cx
+                                           && i_width != pVSCC->MaxOutputSize.cx)
+                                       ||
+                                       (   pVSCC->OutputGranularityX
+                                           && ((i_width % pVSCC->OutputGranularityX)
+                                               || pVSCC->MinOutputSize.cx > i_width
+                                               || i_width > pVSCC->MaxOutputSize.cx )))
                                     {
                                         // required width not compatible, try next media type
                                         FreeMediaType( *p_mt );
@@ -1332,9 +1337,14 @@
 
                                 if( i_height )
                                 {
-                                    if( i_height % pVSCC->OutputGranularityY
-                                     || pVSCC->MinOutputSize.cy > i_height
-                                     || i_height > pVSCC->MaxOutputSize.cy )
+                                    if((   !pVSCC->OutputGranularityY
+                                           && i_height != pVSCC->MinOutputSize.cy
+                                           && i_height != pVSCC->MaxOutputSize.cy)
+                                       ||
+                                       (   pVSCC->OutputGranularityY
+                                           && ((i_height % pVSCC->OutputGranularityY)
+                                               || pVSCC->MinOutputSize.cy > i_height
+                                               || i_height > pVSCC->MaxOutputSize.cy )))
                                     {
                                         // required height not compatible, try next media type
                                         FreeMediaType( *p_mt );
@@ -1375,9 +1385,14 @@
                                     if( ! val )
                                         val = 2;
 
-                                    if( val % pASCC->ChannelsGranularity
-                                     || (unsigned int)val < pASCC->MinimumChannels
-                                     || (unsigned int)val > pASCC->MaximumChannels )
+                                    if( (   !pASCC->ChannelsGranularity
+                                            && (unsigned int)val != pASCC->MinimumChannels
+                                            && (unsigned int)val != pASCC->MaximumChannels)
+                                        ||
+                                        (   pASCC->ChannelsGranularity
+                                            && ((val % pASCC->ChannelsGranularity)
+                                                || (unsigned int)val < pASCC->MinimumChannels
+                                                || (unsigned int)val > pASCC->MaximumChannels)))
                                     {
                                         // required number channels not available, try next media type
                                         FreeMediaType( *p_mt );
@@ -1390,9 +1405,14 @@
                                     if( ! val )
                                         val = 44100;
 
-                                    if( val % pASCC->SampleFrequencyGranularity
-                                     || (unsigned int)val < pASCC->MinimumSampleFrequency
-                                     || (unsigned int)val > pASCC->MaximumSampleFrequency )
+                                    if( (   !pASCC->SampleFrequencyGranularity
+                                            && (unsigned int)val != pASCC->MinimumSampleFrequency
+                                            && (unsigned int)val != pASCC->MaximumSampleFrequency)
+                                        ||
+                                        (   pASCC->SampleFrequencyGranularity
+                                            && ((val % pASCC->SampleFrequencyGranularity)
+                                                || (unsigned int)val < pASCC->MinimumSampleFrequency
+                                                || (unsigned int)val > pASCC->MaximumSampleFrequency )))
                                     {
                                         // required sampling rate not available, try next media type
                                         FreeMediaType( *p_mt );
@@ -1410,9 +1430,14 @@
                                             val = 16;
                                     }
 
-                                    if( val % pASCC->BitsPerSampleGranularity
-                                     || (unsigned int)val < pASCC->MinimumBitsPerSample
-                                     || (unsigned int)val > pASCC->MaximumBitsPerSample )
+                                    if( (   !pASCC->BitsPerSampleGranularity
+                                            && (unsigned int)val != pASCC->MinimumBitsPerSample
+                                            &&	(unsigned int)val != pASCC->MaximumBitsPerSample )
+                                        ||
+                                        (   pASCC->BitsPerSampleGranularity
+                                            && ((val % pASCC->BitsPerSampleGranularity)
+                                                || (unsigned int)val < pASCC->MinimumBitsPerSample
+                                                || (unsigned int)val > pASCC->MaximumBitsPerSample )))
                                     {
                                         // required sample size not available, try next media type
                                         FreeMediaType( *p_mt );