[vlc-commits] remoteosd: fix gcry_ciper_open() error handling

[Zhouyang]

vlc/vlc-2.2 | branch: master | Zhouyang <jiazhouyang09 at gmail.com> | Mon Aug  7 22:31:19 2017 +0800| [ee69f6c27e4261ec60eede9301e4857ad4852426] | committer: Rémi Denis-Courmont

remoteosd: fix gcry_ciper_open() error handling

Fixes #18656.

Signed-off-by: Rémi Denis-Courmont <remi at remlab.net>

> http://git.videolan.org/gitweb.cgi/vlc/vlc-2.2.git/?a=commit;h=ee69f6c27e4261ec60eede9301e4857ad4852426
---

 modules/video_filter/remoteosd.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/modules/video_filter/remoteosd.c b/modules/video_filter/remoteosd.c
index 4b73e21ff1..49c0fb625f 100644
--- a/modules/video_filter/remoteosd.c
+++ b/modules/video_filter/remoteosd.c
@@ -188,7 +188,7 @@ static inline bool raw_line(  filter_sys_t* p_sys,
                               uint16_t i_x, uint16_t i_y,
                               uint16_t i_w );
 
-static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
+static int vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
 
 
 /*****************************************************************************
@@ -469,7 +469,11 @@ static bool handshaking ( filter_t *p_filter )
             return false;
         }
 
-        vnc_encrypt_bytes( challenge, p_sys->psz_passwd );
+        if ( vnc_encrypt_bytes( challenge, p_sys->psz_passwd ) != VLC_SUCCESS)
+        {
+            msg_Err( p_filter, "Could not encrypt password challenge" );
+            return false;
+        }
 
         if( !write_exact(p_filter, p_sys->i_socket,
                          (char*)challenge, CHALLENGESIZE ) )
@@ -1445,7 +1449,7 @@ static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
     return VLC_SUCCESS;
 }
 
-static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
+static int vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
 {
     unsigned char key[8];
     unsigned int i;
@@ -1454,7 +1458,10 @@ static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
         key[i] = i < strlen( passwd ) ? passwd[i] : '\0';
 
     gcry_cipher_hd_t ctx;
-    gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
+    int gcrypt_err = gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
+    if (gcrypt_err != 0){
+        return VLC_EGENERIC;
+    }
 
     /* reverse bits of the key */
     for( i = 0 ; i < 8 ; i ++ )
@@ -1471,5 +1478,6 @@ static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
     gcry_cipher_setkey( ctx, key, 8 );
     gcry_cipher_encrypt( ctx, bytes, CHALLENGESIZE, bytes, CHALLENGESIZE );
     gcry_cipher_close( ctx );
+    return VLC_SUCCESS;
 }