[x265] [PATCH STABLE V2] api: move recursion check to just before generic libx265 load (fixes #171)

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1440042668 -19800
#      Thu Aug 20 09:21:08 2015 +0530
# Branch stable
# Node ID 3c33c8db564c0173c6924b2ce3c58a8dd5255a20
# Parent  db1ea3ea849a44ff3a0a8ce0ef38951cbcb49fb0
api: move recursion check to just before generic libx265 load (fixes #171)

diff -r db1ea3ea849a -r 3c33c8db564c source/encoder/api.cpp
--- a/source/encoder/api.cpp	Thu Aug 20 10:20:20 2015 +0530
+++ b/source/encoder/api.cpp	Thu Aug 20 09:21:08 2015 +0530
@@ -342,11 +342,6 @@
 {
     if (bitDepth && bitDepth != X265_DEPTH)
     {
-        if (g_recursion)
-            return NULL;
-        else
-            g_recursion = 1;
-
 #if LINKED_8BIT
         if (bitDepth == 8) return x265_8bit::x265_api_get(0);
 #endif
@@ -373,6 +368,11 @@
         const x265_api* api = NULL;
         int reqDepth = 0;
 
+        if (g_recursion > 1)
+            return NULL;
+        else
+            g_recursion++;
+
 #if _WIN32
         HMODULE h = LoadLibraryA(libname);
         if (!h)
@@ -401,6 +401,8 @@
         }
 #endif
 
+        g_recursion--;
+
         if (api && bitDepth != api->bit_depth)
         {
             x265_log(NULL, X265_LOG_WARNING, "%s does not support requested bitDepth %d\n", libname, bitDepth);
@@ -422,14 +424,6 @@
         return NULL;
     }
 
-    if (g_recursion)
-    {
-        if (err) *err = X265_API_QUERY_ERR_LIB_NOT_FOUND;
-        return NULL;
-    }
-    else
-        g_recursion = 1;
-
     if (err) *err = X265_API_QUERY_ERR_NONE;
 
     if (bitDepth && bitDepth != X265_DEPTH)
@@ -464,6 +458,14 @@
         int reqDepth = 0;
         int e = X265_API_QUERY_ERR_LIB_NOT_FOUND;
 
+        if (g_recursion > 1)
+        {
+            if (err) *err = X265_API_QUERY_ERR_LIB_NOT_FOUND;
+            return NULL;
+        }
+        else
+            g_recursion++;
+
 #if _WIN32
         HMODULE h = LoadLibraryA(libname);
         if (!h)
@@ -494,6 +496,8 @@
         }
 #endif
 
+        g_recursion--;
+
         if (api && bitDepth != api->bit_depth)
         {
             x265_log(NULL, X265_LOG_WARNING, "%s does not support requested bitDepth %d\n", libname, bitDepth);