[x265] [PATCH 1 of 2] api: move recursion check to just before generic libx265 load (fixes #171)

[Steve Borho]

# HG changeset patch
# User Steve Borho <steve at borho.org>
# Date 1440042668 -19800
#      Thu Aug 20 09:21:08 2015 +0530
# Branch stable
# Node ID 6fb3f94d24b563d60b401d0dae9bc090f99546c0
# Parent  b4ea62725219c3c8a784a33998d97147d763f603
api: move recursion check to just before generic libx265 load (fixes #171)

Note that we allow more than one level of recursion before aborting, because
apps like ffmpeg will probe for supported bit-depths before picking their
final depth.  Picking some random number 6 out of the air kind of sucks, but I
can't think anything better.  Apps which will create many encoders with
different bit depths are suggested to cache the API pointer for each depth.

diff -r b4ea62725219 -r 6fb3f94d24b5 doc/reST/api.rst
--- a/doc/reST/api.rst	Wed Aug 19 13:59:38 2015 +0530
+++ b/doc/reST/api.rst	Thu Aug 20 09:21:08 2015 +0530
@@ -450,6 +450,14 @@
 sizeof(x265_parm) and thereby ignore changes to that structure (which
 account for a large percentage of X265_BUILD bumps).
 
+.. Note::
+
+    Long running applications which perform multiple encodes at varying
+    bit depths are suggested to query the per-bit-depth API structures a
+    single time and to cache the returned x265_api pointers.  In certain
+    build configurations, repeated calls to the introspection methods
+    may begin returning NULL.
+
 Build Implications
 ------------------
 
diff -r b4ea62725219 -r 6fb3f94d24b5 source/encoder/api.cpp
--- a/source/encoder/api.cpp	Wed Aug 19 13:59:38 2015 +0530
+++ b/source/encoder/api.cpp	Thu Aug 20 09:21:08 2015 +0530
@@ -342,11 +342,6 @@
 {
     if (bitDepth && bitDepth != X265_DEPTH)
     {
-        if (g_recursion)
-            return NULL;
-        else
-            g_recursion = 1;
-
 #if LINKED_8BIT
         if (bitDepth == 8) return x265_8bit::x265_api_get(0);
 #endif
@@ -377,6 +372,11 @@
         HMODULE h = LoadLibraryA(libname);
         if (!h)
         {
+            if (g_recursion > 6)
+                return NULL;
+            else
+                g_recursion++;
+
             h = LoadLibraryA(multilibname);
             reqDepth = bitDepth;
         }
@@ -390,6 +390,11 @@
         void* h = dlopen(libname, RTLD_LAZY | RTLD_LOCAL);
         if (!h)
         {
+            if (g_recursion > 6)
+                return NULL;
+            else
+                g_recursion++;
+
             h = dlopen(multilibname, RTLD_LAZY | RTLD_LOCAL);
             reqDepth = bitDepth;
         }
@@ -422,14 +427,6 @@
         return NULL;
     }
 
-    if (g_recursion)
-    {
-        if (err) *err = X265_API_QUERY_ERR_LIB_NOT_FOUND;
-        return NULL;
-    }
-    else
-        g_recursion = 1;
-
     if (err) *err = X265_API_QUERY_ERR_NONE;
 
     if (bitDepth && bitDepth != X265_DEPTH)
@@ -468,6 +465,14 @@
         HMODULE h = LoadLibraryA(libname);
         if (!h)
         {
+            if (g_recursion > 6)
+            {
+                if (err) *err = X265_API_QUERY_ERR_LIB_NOT_FOUND;
+                return NULL;
+            }
+            else
+                g_recursion++;
+
             h = LoadLibraryA(multilibname);
             reqDepth = bitDepth;
         }
@@ -482,6 +487,14 @@
         void* h = dlopen(libname, RTLD_LAZY | RTLD_LOCAL);
         if (!h)
         {
+            if (g_recursion > 6)
+            {
+                if (err) *err = X265_API_QUERY_ERR_LIB_NOT_FOUND;
+                return NULL;
+            }
+            else
+                g_recursion++;
+
             h = dlopen(multilibname, RTLD_LAZY | RTLD_LOCAL);
             reqDepth = bitDepth;
         }