[x265] [PATCH 1 of 2] api: move recursion check to just before generic libx265 load (fixes #171)

[Steve Borho]

On 08/20, Steve Borho wrote:
> # HG changeset patch
> # User Steve Borho <steve at borho.org>
> # Date 1440042668 -19800
> #      Thu Aug 20 09:21:08 2015 +0530
> # Branch stable
> # Node ID 6fb3f94d24b563d60b401d0dae9bc090f99546c0
> # Parent  b4ea62725219c3c8a784a33998d97147d763f603
> api: move recursion check to just before generic libx265 load (fixes #171)
> 
> Note that we allow more than one level of recursion before aborting, because
> apps like ffmpeg will probe for supported bit-depths before picking their
> final depth.  Picking some random number 6 out of the air kind of sucks, but I
> can't think anything better.  Apps which will create many encoders with
> different bit depths are suggested to cache the API pointer for each depth.

ignore this version of the patch, this would be safe if we simply
decremented the recursion count on returning from the api query call

> diff -r b4ea62725219 -r 6fb3f94d24b5 doc/reST/api.rst
> --- a/doc/reST/api.rst	Wed Aug 19 13:59:38 2015 +0530
> +++ b/doc/reST/api.rst	Thu Aug 20 09:21:08 2015 +0530
> @@ -450,6 +450,14 @@
>  sizeof(x265_parm) and thereby ignore changes to that structure (which
>  account for a large percentage of X265_BUILD bumps).
>  
> +.. Note::
> +
> +    Long running applications which perform multiple encodes at varying
> +    bit depths are suggested to query the per-bit-depth API structures a
> +    single time and to cache the returned x265_api pointers.  In certain
> +    build configurations, repeated calls to the introspection methods
> +    may begin returning NULL.
> +
>  Build Implications
>  ------------------
>  
> diff -r b4ea62725219 -r 6fb3f94d24b5 source/encoder/api.cpp
> --- a/source/encoder/api.cpp	Wed Aug 19 13:59:38 2015 +0530
> +++ b/source/encoder/api.cpp	Thu Aug 20 09:21:08 2015 +0530
> @@ -342,11 +342,6 @@
>  {
>      if (bitDepth && bitDepth != X265_DEPTH)
>      {
> -        if (g_recursion)
> -            return NULL;
> -        else
> -            g_recursion = 1;
> -
>  #if LINKED_8BIT
>          if (bitDepth == 8) return x265_8bit::x265_api_get(0);
>  #endif
> @@ -377,6 +372,11 @@
>          HMODULE h = LoadLibraryA(libname);
>          if (!h)
>          {
> +            if (g_recursion > 6)
> +                return NULL;
> +            else
> +                g_recursion++;
> +
>              h = LoadLibraryA(multilibname);
>              reqDepth = bitDepth;
>          }
> @@ -390,6 +390,11 @@
>          void* h = dlopen(libname, RTLD_LAZY | RTLD_LOCAL);
>          if (!h)
>          {
> +            if (g_recursion > 6)
> +                return NULL;
> +            else
> +                g_recursion++;
> +
>              h = dlopen(multilibname, RTLD_LAZY | RTLD_LOCAL);
>              reqDepth = bitDepth;
>          }
> @@ -422,14 +427,6 @@
>          return NULL;
>      }
>  
> -    if (g_recursion)
> -    {
> -        if (err) *err = X265_API_QUERY_ERR_LIB_NOT_FOUND;
> -        return NULL;
> -    }
> -    else
> -        g_recursion = 1;
> -
>      if (err) *err = X265_API_QUERY_ERR_NONE;
>  
>      if (bitDepth && bitDepth != X265_DEPTH)
> @@ -468,6 +465,14 @@
>          HMODULE h = LoadLibraryA(libname);
>          if (!h)
>          {
> +            if (g_recursion > 6)
> +            {
> +                if (err) *err = X265_API_QUERY_ERR_LIB_NOT_FOUND;
> +                return NULL;
> +            }
> +            else
> +                g_recursion++;
> +
>              h = LoadLibraryA(multilibname);
>              reqDepth = bitDepth;
>          }
> @@ -482,6 +487,14 @@
>          void* h = dlopen(libname, RTLD_LAZY | RTLD_LOCAL);
>          if (!h)
>          {
> +            if (g_recursion > 6)
> +            {
> +                if (err) *err = X265_API_QUERY_ERR_LIB_NOT_FOUND;
> +                return NULL;
> +            }
> +            else
> +                g_recursion++;
> +
>              h = dlopen(multilibname, RTLD_LAZY | RTLD_LOCAL);
>              reqDepth = bitDepth;
>          }

-- 
Steve Borho