[vlc-devel] [PATCH] vout: vulkan: add support for custom user shaders

[Niklas Haas]

From: Niklas Haas <git at haasn.xyz>

It's finally time to use RAVU/FSRCNN(X)/NNEDI3/Anime4K in VLC.

Currently limited to only one shader, which isn't a huge limitation
seeing as you can just `cat` together as many shaders as you want to use
into one big file. But patches welcome(tm).

In theory it would also be cool if we could just make it a big textbox,
but I couldn't figure out how.
---
 modules/video_output/placebo_utils.h  |  3 ++
 modules/video_output/vulkan/display.c | 74 +++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)

diff --git a/modules/video_output/placebo_utils.h b/modules/video_output/placebo_utils.h
index 99c3f318b5..7e460093a2 100644
--- a/modules/video_output/placebo_utils.h
+++ b/modules/video_output/placebo_utils.h
@@ -496,4 +496,7 @@ static const struct pl_filter_function *const filter_fun[] = {
 #define DELAYED_PEAK_TEXT "Allow delaying peak detection by up to one frame"
 #define DELAYED_PEAK_LONGTEXT "In some cases, peak detection may be more convenient to compute if the results are delayed by a frame. When this option is disabled, libplacebo will use an indirect buffer simply to force peak detection results to be up-to-date. Enabling it allows skipping this indirection in order to improve performance at the cost of some potentially noticeable brightness flickering immediately after a scene change."
 
+#define USER_SHADER_FILE_TEXT N_("Custom shader")
+#define USER_SHADER_FILE_LONGTEXT N_("Path to a file containing a custom user shader, in mpv .hook format.")
+
 #endif // VLC_PLACEBO_UTILS_H
diff --git a/modules/video_output/vulkan/display.c b/modules/video_output/vulkan/display.c
index 174627d67d..1ae193fb31 100644
--- a/modules/video_output/vulkan/display.c
+++ b/modules/video_output/vulkan/display.c
@@ -30,6 +30,7 @@
 #include <vlc_common.h>
 #include <vlc_plugin.h>
 #include <vlc_vout_display.h>
+#include <vlc_fs.h>
 
 #include "../placebo_utils.h"
 #include "instance.h"
@@ -65,6 +66,10 @@ struct vout_display_sys_t
     struct pl_color_space target;
 #if PL_API_VER >= 13
     struct pl_peak_detect_params peak_detect;
+#endif
+#if PL_API_VER >= 58
+    const struct pl_hook *hook;
+    char *hook_path;
 #endif
     enum pl_chroma_location yuv_chroma_loc;
     int dither_depth;
@@ -170,6 +175,9 @@ static void Close(vout_display_t *vd)
         free(sys->overlay_tex);
     }
 
+    pl_mpv_user_shader_destroy(&sys->hook);
+    free(sys->hook_path);
+
     pl_renderer_destroy(&sys->renderer);
 
     vlc_vk_Release(sys->vk);
@@ -361,6 +369,57 @@ static int Control(vout_display_t *vd, int query, va_list ap)
     return VLC_EGENERIC;
 }
 
+#if PL_API_VER >= 58
+
+static void load_user_shader(vout_display_sys_t *sys, const char *filepath)
+{
+    if (!filepath || !*filepath) {
+        pl_mpv_user_shader_destroy(&sys->hook);
+        return;
+    }
+
+    if (sys->hook_path && strcmp(filepath, sys->hook_path) == 0)
+        return; // same shader
+
+    const struct pl_gpu *gpu = sys->vk->vulkan->gpu;
+    char *shader_str = NULL;
+    FILE *fs = NULL;
+
+    free(sys->hook_path);
+    sys->hook_path = strdup(filepath);
+
+    fs = vlc_fopen(filepath, "rb");
+    int ret = fseek(fs, 0, SEEK_END);
+    if (ret == -1)
+        goto error;
+    long length = ftell(fs);
+    if (length == -1 || length >= 1024*1024) // 1 MB
+        goto error;
+    rewind(fs);
+
+    shader_str = vlc_alloc(length, sizeof(*shader_str));
+    if (!shader_str)
+        goto error;
+    ret = fread(shader_str, length, 1, fs);
+    if (ret != 1)
+        goto error;
+    sys->hook = pl_mpv_user_shader_parse(gpu, shader_str, length);
+    if (!sys->hook)
+        goto error;
+
+    fclose(fs);
+    free(shader_str);
+    return;
+
+error:
+    free(shader_str);
+    if (fs)
+        fclose(fs);
+    return;
+}
+
+#endif // PL_API_VER >= 58
+
 // Options
 
 #define VK_TEXT N_("Vulkan surface extension")
@@ -379,6 +438,11 @@ vlc_module_begin ()
     add_shortcut ("vulkan", "vk")
     add_module ("vk", "vulkan", NULL, VK_TEXT, PROVIDER_LONGTEXT)
 
+#if PL_API_VER >= 58
+    set_section("Custom shaders", NULL)
+    add_loadfile("user-shader-file", NULL, USER_SHADER_FILE_TEXT, USER_SHADER_FILE_LONGTEXT)
+#endif
+
     set_section("Scaling", NULL)
     add_integer("upscaler-preset", SCALE_BUILTIN,
             UPSCALER_PRESET_TEXT, SCALER_PRESET_LONGTEXT, false)
@@ -628,4 +692,14 @@ static void UpdateParams(vout_display_t *vd)
         .transfer = var_InheritInteger(vd, "target-trc"),
         .sig_avg = var_InheritFloat(vd, "target-avg"),
     };
+
+#if PL_API_VER >= 58
+    load_user_shader(sys, var_InheritString(vd, "user-shader-file"));
+    if (sys->hook) {
+        sys->params.hooks = &sys->hook;
+        sys->params.num_hooks = 1;
+    } else {
+        sys->params.num_hooks = 0;
+    }
+#endif
 }
-- 
2.26.2