[vlc-devel] commit: dshow: prevent division by zero (Edouard Gomez )

[git version control]

vlc | branch: 0.9-bugfix | Edouard Gomez <ed.gomez at free.fr> | Wed Oct 15 00:04:27 2008 +0200| [b63538354a6a49ae5a878edd37221480cb7850f5] | committer: Jean-Baptiste Kempf 

dshow: prevent division by zero

# HG changeset patch
# User Edouard Gomez <ed.gomez at free.fr>
# Date 1224021633 -7200
# Node ID 9564c1736b76c96f37fb85476d36496b3d7955bb
# Parent  40d5c2186f7fb31e944b5b542b3366bbc420cc0e
dshow: prevent division by zero

In the DShow capture module, division by zero is possible in
multiple modulo statements when dealing with granularity values
returned by capture filters.

Change the code so that:
 - if granularity is zero, then width/height/samplingrate must be
   a perfect match with requested settings.
 - if granularity is non zero, then use the old condition tests.

This fixes real crashes reported by VLC users on different forums.

See:
http://forum.doom9.org/archive/index.php/t-134655.html
http://forum.videolan.org/viewtopic.php?f=2&t=37056&p=114065&hilit=blackmagic

Tested with both a Decklink HD Extreme card and a Decklink HD Pro card.

Signed-off-by: Jean-Baptiste Kempf <jb at videolan.org>
(cherry picked from commit 0dd04d073d5f4c653e03064273985ce51cebd43c)

Signed-off-by: Jean-Baptiste Kempf <jb at videolan.org>

> http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=b63538354a6a49ae5a878edd37221480cb7850f5
---

 modules/access/dshow/dshow.cpp |   55 +++++++++++++++++++++++++++++-----------
 1 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/modules/access/dshow/dshow.cpp b/modules/access/dshow/dshow.cpp
index c6dd19b..dbbe31b 100644
--- a/modules/access/dshow/dshow.cpp
+++ b/modules/access/dshow/dshow.cpp
@@ -1318,9 +1318,14 @@ static size_t EnumDeviceCaps( vlc_object_t *p_this, IBaseFilter *p_filter,
 
                                 if( i_width )
                                 {
-                                    if( i_width % pVSCC->OutputGranularityX
-                                     || pVSCC->MinOutputSize.cx > i_width
-                                     || i_width > pVSCC->MaxOutputSize.cx )
+                                    if((   !pVSCC->OutputGranularityX
+                                           && i_width != pVSCC->MinOutputSize.cx
+                                           && i_width != pVSCC->MaxOutputSize.cx)
+                                       ||
+                                       (   pVSCC->OutputGranularityX
+                                           && ((i_width % pVSCC->OutputGranularityX)
+                                               || pVSCC->MinOutputSize.cx > i_width
+                                               || i_width > pVSCC->MaxOutputSize.cx )))
                                     {
                                         // required width not compatible, try next media type
                                         FreeMediaType( *p_mt );
@@ -1332,9 +1337,14 @@ static size_t EnumDeviceCaps( vlc_object_t *p_this, IBaseFilter *p_filter,
 
                                 if( i_height )
                                 {
-                                    if( i_height % pVSCC->OutputGranularityY
-                                     || pVSCC->MinOutputSize.cy > i_height
-                                     || i_height > pVSCC->MaxOutputSize.cy )
+                                    if((   !pVSCC->OutputGranularityY
+                                           && i_height != pVSCC->MinOutputSize.cy
+                                           && i_height != pVSCC->MaxOutputSize.cy)
+                                       ||
+                                       (   pVSCC->OutputGranularityY
+                                           && ((i_height % pVSCC->OutputGranularityY)
+                                               || pVSCC->MinOutputSize.cy > i_height
+                                               || i_height > pVSCC->MaxOutputSize.cy )))
                                     {
                                         // required height not compatible, try next media type
                                         FreeMediaType( *p_mt );
@@ -1375,9 +1385,14 @@ static size_t EnumDeviceCaps( vlc_object_t *p_this, IBaseFilter *p_filter,
                                     if( ! val )
                                         val = 2;
 
-                                    if( val % pASCC->ChannelsGranularity
-                                     || (unsigned int)val < pASCC->MinimumChannels
-                                     || (unsigned int)val > pASCC->MaximumChannels )
+                                    if( (   !pASCC->ChannelsGranularity
+                                            && (unsigned int)val != pASCC->MinimumChannels
+                                            && (unsigned int)val != pASCC->MaximumChannels)
+                                        ||
+                                        (   pASCC->ChannelsGranularity
+                                            && ((val % pASCC->ChannelsGranularity)
+                                                || (unsigned int)val < pASCC->MinimumChannels
+                                                || (unsigned int)val > pASCC->MaximumChannels)))
                                     {
                                         // required number channels not available, try next media type
                                         FreeMediaType( *p_mt );
@@ -1390,9 +1405,14 @@ static size_t EnumDeviceCaps( vlc_object_t *p_this, IBaseFilter *p_filter,
                                     if( ! val )
                                         val = 44100;
 
-                                    if( val % pASCC->SampleFrequencyGranularity
-                                     || (unsigned int)val < pASCC->MinimumSampleFrequency
-                                     || (unsigned int)val > pASCC->MaximumSampleFrequency )
+                                    if( (   !pASCC->SampleFrequencyGranularity
+                                            && (unsigned int)val != pASCC->MinimumSampleFrequency
+                                            && (unsigned int)val != pASCC->MaximumSampleFrequency)
+                                        ||
+                                        (   pASCC->SampleFrequencyGranularity
+                                            && ((val % pASCC->SampleFrequencyGranularity)
+                                                || (unsigned int)val < pASCC->MinimumSampleFrequency
+                                                || (unsigned int)val > pASCC->MaximumSampleFrequency )))
                                     {
                                         // required sampling rate not available, try next media type
                                         FreeMediaType( *p_mt );
@@ -1410,9 +1430,14 @@ static size_t EnumDeviceCaps( vlc_object_t *p_this, IBaseFilter *p_filter,
                                             val = 16;
                                     }
 
-                                    if( val % pASCC->BitsPerSampleGranularity
-                                     || (unsigned int)val < pASCC->MinimumBitsPerSample
-                                     || (unsigned int)val > pASCC->MaximumBitsPerSample )
+                                    if( (   !pASCC->BitsPerSampleGranularity
+                                            && (unsigned int)val != pASCC->MinimumBitsPerSample
+                                            &&	(unsigned int)val != pASCC->MaximumBitsPerSample )
+                                        ||
+                                        (   pASCC->BitsPerSampleGranularity
+                                            && ((val % pASCC->BitsPerSampleGranularity)
+                                                || (unsigned int)val < pASCC->MinimumBitsPerSample
+                                                || (unsigned int)val > pASCC->MaximumBitsPerSample )))
                                     {
                                         // required sample size not available, try next media type
                                         FreeMediaType( *p_mt );