[vlc-devel] [PATCH 1 of 4] dshow: prevent division by zero

[Edouard Gomez]

# HG changeset patch
# User Edouard Gomez <ed.gomez at free.fr>
# Date 1224021633 -7200
# Node ID 9564c1736b76c96f37fb85476d36496b3d7955bb
# Parent  40d5c2186f7fb31e944b5b542b3366bbc420cc0e
dshow: prevent division by zero

In the DShow capture module, division by zero is possible in
multiple modulo statements when dealing with granularity values
returned by capture filters.

Change the code so that:
 - if granularity is zero, then width/height/samplingrate must be
   a perfect match with requested settings.
 - if granularity is non zero, then use the old condition tests.

This fixes real crashes reported by VLC users on different forums.

See:
http://forum.doom9.org/archive/index.php/t-134655.html
http://forum.videolan.org/viewtopic.php?f=2&t=37056&p=114065&hilit=blackmagic

Tested with both a Decklink HD Extreme card and a Decklink HD Pro card.

diff --git a/modules/access/dshow/dshow.cpp b/modules/access/dshow/dshow.cpp
--- a/modules/access/dshow/dshow.cpp
+++ b/modules/access/dshow/dshow.cpp
@@ -1318,9 +1318,14 @@
 
                                 if( i_width )
                                 {
-                                    if( i_width % pVSCC->OutputGranularityX
-                                     || pVSCC->MinOutputSize.cx > i_width
-                                     || i_width > pVSCC->MaxOutputSize.cx )
+                                    if((   !pVSCC->OutputGranularityX
+                                           && i_width != pVSCC->MinOutputSize.cx
+                                           && i_width != pVSCC->MaxOutputSize.cx)
+                                       ||
+                                       (   pVSCC->OutputGranularityX
+                                           && ((i_width % pVSCC->OutputGranularityX)
+                                               || pVSCC->MinOutputSize.cx > i_width
+                                               || i_width > pVSCC->MaxOutputSize.cx )))
                                     {
                                         // required width not compatible, try next media type
                                         FreeMediaType( *p_mt );
@@ -1332,9 +1337,14 @@
 
                                 if( i_height )
                                 {
-                                    if( i_height % pVSCC->OutputGranularityY
-                                     || pVSCC->MinOutputSize.cy > i_height
-                                     || i_height > pVSCC->MaxOutputSize.cy )
+                                    if((   !pVSCC->OutputGranularityY
+                                           && i_height != pVSCC->MinOutputSize.cy
+                                           && i_height != pVSCC->MaxOutputSize.cy)
+                                       ||
+                                       (   pVSCC->OutputGranularityY
+                                           && ((i_height % pVSCC->OutputGranularityY)
+                                               || pVSCC->MinOutputSize.cy > i_height
+                                               || i_height > pVSCC->MaxOutputSize.cy )))
                                     {
                                         // required height not compatible, try next media type
                                         FreeMediaType( *p_mt );
@@ -1375,9 +1385,14 @@
                                     if( ! val )
                                         val = 2;
 
-                                    if( val % pASCC->ChannelsGranularity
-                                     || (unsigned int)val < pASCC->MinimumChannels
-                                     || (unsigned int)val > pASCC->MaximumChannels )
+                                    if( (   !pASCC->ChannelsGranularity
+                                            && (unsigned int)val != pASCC->MinimumChannels
+                                            && (unsigned int)val != pASCC->MaximumChannels)
+                                        ||
+                                        (   pASCC->ChannelsGranularity
+                                            && ((val % pASCC->ChannelsGranularity)
+                                                || (unsigned int)val < pASCC->MinimumChannels
+                                                || (unsigned int)val > pASCC->MaximumChannels)))
                                     {
                                         // required number channels not available, try next media type
                                         FreeMediaType( *p_mt );
@@ -1390,9 +1405,14 @@
                                     if( ! val )
                                         val = 44100;
 
-                                    if( val % pASCC->SampleFrequencyGranularity
-                                     || (unsigned int)val < pASCC->MinimumSampleFrequency
-                                     || (unsigned int)val > pASCC->MaximumSampleFrequency )
+                                    if( (   !pASCC->SampleFrequencyGranularity
+                                            && (unsigned int)val != pASCC->MinimumSampleFrequency
+                                            && (unsigned int)val != pASCC->MaximumSampleFrequency)
+                                        ||
+                                        (   pASCC->SampleFrequencyGranularity
+                                            && ((val % pASCC->SampleFrequencyGranularity)
+                                                || (unsigned int)val < pASCC->MinimumSampleFrequency
+                                                || (unsigned int)val > pASCC->MaximumSampleFrequency )))
                                     {
                                         // required sampling rate not available, try next media type
                                         FreeMediaType( *p_mt );
@@ -1410,9 +1430,14 @@
                                             val = 16;
                                     }
 
-                                    if( val % pASCC->BitsPerSampleGranularity
-                                     || (unsigned int)val < pASCC->MinimumBitsPerSample
-                                     || (unsigned int)val > pASCC->MaximumBitsPerSample )
+                                    if( (   !pASCC->BitsPerSampleGranularity
+                                            && (unsigned int)val != pASCC->MinimumBitsPerSample
+                                            &&	(unsigned int)val != pASCC->MaximumBitsPerSample )
+                                        ||
+                                        (   pASCC->BitsPerSampleGranularity
+                                            && ((val % pASCC->BitsPerSampleGranularity)
+                                                || (unsigned int)val < pASCC->MinimumBitsPerSample
+                                                || (unsigned int)val > pASCC->MaximumBitsPerSample )))
                                     {
                                         // required sample size not available, try next media type
                                         FreeMediaType( *p_mt );