[vlc-devel] [PATCH 2/2] vout: spu: prerender text

Francois Cartegnie fcvlcdev at free.fr
Wed Jul 10 09:57:40 CEST 2019


Starts doing text-renderer work in a separated thread once
the vout dimensions are known.

- Pushed to the thread queue when pushed to spu channel
- Thread worked starts on first SpuSelect, when we know vout size
- Thread processes spu queues by time (sorted when enqueing jobs)
- Tricky: Thread walks its queue for the lowest time, even if it 
   was sorted (the spu times can be changed on the fly by delay adjustment)
- SpuSelect syncs() with the thread and waits for uncompleted spus
- If the picture needs update, this is still done after spuselect (same as 
  before). Need more work, as changing this makes things far more complex.

---
 src/video_output/vout_subpictures.c | 352 ++++++++++++++++++++++------
 1 file changed, 278 insertions(+), 74 deletions(-)

diff --git a/src/video_output/vout_subpictures.c b/src/video_output/vout_subpictures.c
index 7a77eec55d..86611e55bb 100644
--- a/src/video_output/vout_subpictures.c
+++ b/src/video_output/vout_subpictures.c
@@ -69,6 +69,7 @@ struct spu_channel {
 };
 
 typedef struct VLC_VECTOR(struct spu_channel) spu_channel_vector;
+typedef struct VLC_VECTOR(subpicture_t *) spu_prerender_vector;
 
 struct spu_private_t {
     vlc_mutex_t  lock;            /* lock to protect all followings fields */
@@ -106,12 +107,26 @@ struct spu_private_t {
     char           *filter_chain_update;
     vlc_mutex_t    filter_chain_lock;
     filter_chain_t *filter_chain;
+    /**/
+    vlc_thread_t    prerender;
+    vlc_mutex_t     prerender_lock;
+    vlc_cond_t      prerender_cond;
+    vlc_cond_t      prerender_output_cond;
+    spu_prerender_vector prerender_vector;
+    subpicture_t   *p_processed;
+    video_format_t  prerender_fmtsrc;
+    video_format_t  prerender_fmtdst;
+    const vlc_fourcc_t *prerender_chroma_list;
 
     /* */
     vlc_tick_t          last_sort_date;
     vout_thread_t       *vout;
 };
 
+/* forward decl */
+static void spu_PrerenderSync(spu_private_t *, const subpicture_t *);
+static void spu_PrerenderCancel(spu_private_t *, const subpicture_t *);
+
 static void spu_channel_Init(struct spu_channel *channel, size_t id,
                              enum vlc_vout_order order, vlc_clock_t *clock)
 {
@@ -144,11 +159,12 @@ static void spu_channel_DeleteAt(struct spu_channel *channel, size_t index)
     vlc_vector_remove(&channel->entries, index);
 }
 
-static void spu_channel_Clean(struct spu_channel *channel)
+static void spu_channel_Clean(spu_private_t *sys, struct spu_channel *channel)
 {
     for (size_t i = 0; i < channel->entries.size; i++)
     {
         assert(channel->entries.data[i].subpic);
+        spu_PrerenderCancel(sys, channel->entries.data[i].subpic);
         subpicture_Delete(channel->entries.data[i].subpic);
     }
     vlc_vector_destroy(&channel->entries);
@@ -301,6 +317,16 @@ static void SpuRenderText(spu_t *spu,
 
     assert(region->fmt.i_chroma == VLC_CODEC_TEXT);
 
+    //assume rendered text is in sRGB if nothing is set
+    if (region->fmt.transfer == TRANSFER_FUNC_UNDEF)
+        region->fmt.transfer = TRANSFER_FUNC_SRGB;
+    if (region->fmt.primaries == COLOR_PRIMARIES_UNDEF)
+        region->fmt.primaries = COLOR_PRIMARIES_SRGB;
+    if (region->fmt.space == COLOR_SPACE_UNDEF)
+        region->fmt.space = COLOR_SPACE_SRGB;
+    if (region->fmt.color_range == COLOR_RANGE_UNDEF)
+        region->fmt.color_range = COLOR_RANGE_FULL;
+
     if ( region->p_text )
         text->pf_render(text, region, region, chroma_list);
 }
@@ -719,6 +745,7 @@ spu_SelectSubpictures(spu_t *spu, vlc_tick_t system_now,
 
             if (is_rejeted)
             {
+                spu_PrerenderCancel(sys, current);
                 subpicture_Delete(current);
                 vlc_vector_remove(&channel->entries, index);
             }
@@ -764,15 +791,6 @@ static void SpuRenderRegion(spu_t *spu,
 
     /* Render text region */
     if (region->fmt.i_chroma == VLC_CODEC_TEXT) {
-        // assume rendered text is in sRGB if nothing is set
-        if (region->fmt.transfer == TRANSFER_FUNC_UNDEF)
-            region->fmt.transfer = TRANSFER_FUNC_SRGB;
-        if (region->fmt.primaries == COLOR_PRIMARIES_UNDEF)
-            region->fmt.primaries = COLOR_PRIMARIES_SRGB;
-        if (region->fmt.space == COLOR_SPACE_UNDEF)
-            region->fmt.space = COLOR_SPACE_SRGB;
-        if (region->fmt.color_range == COLOR_RANGE_UNDEF)
-            region->fmt.color_range = COLOR_RANGE_FULL;
 
         SpuRenderText(spu, region, chroma_list);
 
@@ -1373,10 +1391,214 @@ static int SubSourceDelProxyCallbacks(filter_t *filter, void *opaque)
     return VLC_SUCCESS;
 }
 
+static void spu_PrerenderWake(spu_private_t *sys,
+                              const video_format_t *fmt_dst,
+                              const video_format_t *fmt_src,
+                              const vlc_fourcc_t *chroma_list)
+{
+    vlc_mutex_lock(&sys->prerender_lock);
+    if(!video_format_IsSimilar(fmt_dst, &sys->prerender_fmtdst))
+    {
+        video_format_Clean(&sys->prerender_fmtdst);
+        video_format_Copy(&sys->prerender_fmtdst, fmt_dst);
+        printf("FMT src %dx%d dst %dx%d\n",
+               fmt_src->i_visible_width, fmt_src->i_visible_height,
+               fmt_dst->i_visible_width, fmt_dst->i_visible_height);
+    }
+    if(!video_format_IsSimilar(fmt_src, &sys->prerender_fmtsrc))
+    {
+        video_format_Clean(&sys->prerender_fmtsrc);
+        video_format_Copy(&sys->prerender_fmtsrc, fmt_src);
+    }
+    sys->prerender_chroma_list = chroma_list;
+    vlc_cond_signal(&sys->prerender_cond);
+    vlc_mutex_unlock(&sys->prerender_lock);
+}
+
+static void spu_PrerenderEnqueue(spu_private_t *sys, subpicture_t *p_subpic)
+{
+    vlc_mutex_lock(&sys->prerender_lock);
+    subpicture_t *p;
+    size_t i_idx = 0;
+    vlc_vector_foreach(p, &sys->prerender_vector)
+    {
+        if(p->i_start > p_subpic->i_start)
+            break;
+        i_idx++;
+    }
+    vlc_vector_insert(&sys->prerender_vector, i_idx, p_subpic);
+    vlc_mutex_unlock(&sys->prerender_lock);
+}
+
+static void spu_PrerenderCancel(spu_private_t *sys, const subpicture_t *p_subpic)
+{
+    vlc_mutex_lock(&sys->prerender_lock);
+    ssize_t i_idx;
+    vlc_vector_index_of(&sys->prerender_vector, p_subpic, &i_idx);
+    if(i_idx >= 0)
+        vlc_vector_remove(&sys->prerender_vector, i_idx);
+    else while(sys->p_processed == p_subpic)
+        vlc_cond_wait(&sys->prerender_output_cond, &sys->prerender_lock);
+    vlc_mutex_unlock(&sys->prerender_lock);
+}
+
+static void spu_PrerenderSync(spu_private_t *sys, const subpicture_t *p_subpic)
+{
+    vlc_mutex_lock(&sys->prerender_lock);
+    ssize_t i_idx;
+    vlc_vector_index_of(&sys->prerender_vector, p_subpic, &i_idx);
+    while(i_idx >= 0 || sys->p_processed == p_subpic)
+    {
+        vlc_cond_wait(&sys->prerender_output_cond, &sys->prerender_lock);
+        vlc_vector_index_of(&sys->prerender_vector, p_subpic, &i_idx);
+    }
+    vlc_mutex_unlock(&sys->prerender_lock);
+}
+
+static void spu_PrerenderText(spu_t *spu, subpicture_t *p_subpic)
+{
+    spu_private_t *sys = spu->p;
+
+    if (p_subpic->i_original_picture_width  <= 0 ||
+        p_subpic->i_original_picture_height <= 0) {
+        if (p_subpic->i_original_picture_width  > 0 ||
+            p_subpic->i_original_picture_height > 0)
+            msg_Err(spu, "original picture size %dx%d is unsupported",
+                     p_subpic->i_original_picture_width,
+                     p_subpic->i_original_picture_height);
+        else
+            msg_Warn(spu, "original picture size is undefined");
+
+        p_subpic->i_original_picture_width  = sys->prerender_fmtsrc.i_visible_width;
+        p_subpic->i_original_picture_height = sys->prerender_fmtsrc.i_visible_height;
+    }
+
+    /* FIXME aspect ratio ? */
+    sys->text->fmt_out.video.i_width          =
+    sys->text->fmt_out.video.i_visible_width  = p_subpic->i_original_picture_width;
+
+    sys->text->fmt_out.video.i_height         =
+    sys->text->fmt_out.video.i_visible_height = p_subpic->i_original_picture_height;
+
+    subpicture_Update(p_subpic,
+                      &sys->prerender_fmtsrc, &sys->prerender_fmtdst,
+                      p_subpic->b_subtitle ? p_subpic->i_start : vlc_tick_now());
+
+    subpicture_region_t *region;
+    for (region = p_subpic->p_region; region != NULL; region = region->p_next)
+    {
+        if(region->fmt.i_chroma != VLC_CODEC_TEXT)
+            continue;
+        vlc_tick_t t = vlc_tick_now();
+        SpuRenderText(spu, region, sys->prerender_chroma_list);
+        printf("Prerendered in %ld ms %ld %ld\n",
+               (vlc_tick_now() - t) / 1000,
+               p_subpic->i_start, t);
+    }
+}
+
+static void * spu_PrerenderThread(void *priv)
+{
+    spu_t *spu = priv;
+    spu_private_t *sys = spu->p;
+
+    vlc_mutex_lock(&sys->prerender_lock);
+    for( ;; )
+    {
+        mutex_cleanup_push(&sys->prerender_lock);
+        while(sys->prerender_vector.size == 0)
+            vlc_cond_wait(&sys->prerender_cond, &sys->prerender_lock);
+
+        size_t i_idx = 0;
+        sys->p_processed = sys->prerender_vector.data[0];
+        for(size_t i=1; i<sys->prerender_vector.size; i++)
+        {
+             if(sys->p_processed->i_start > sys->prerender_vector.data[i]->i_start)
+             {
+                 sys->p_processed = sys->prerender_vector.data[i];
+                 i_idx = i;
+             }
+        }
+        vlc_vector_remove(&sys->prerender_vector, i_idx);
+        vlc_mutex_unlock(&sys->prerender_lock);
+        vlc_cleanup_pop();
+
+        int canc = vlc_savecancel();
+        spu_PrerenderText(spu, sys->p_processed);
+        vlc_restorecancel(canc);
+
+        vlc_mutex_lock(&sys->prerender_lock);
+        sys->p_processed = NULL;
+        vlc_cond_signal(&sys->prerender_output_cond);
+    }
+
+    return NULL;
+}
+
 /*****************************************************************************
  * Public API
  *****************************************************************************/
 
+static void spu_Cleanup(spu_t *spu)
+{
+    spu_private_t *sys = spu->p;
+
+    if (sys->text)
+        FilterRelease(sys->text);
+
+    if (sys->scale_yuvp)
+        FilterRelease(sys->scale_yuvp);
+
+    if (sys->scale)
+        FilterRelease(sys->scale);
+
+    filter_chain_ForEach(sys->source_chain, SubSourceClean, spu);
+    if (sys->vout)
+        filter_chain_ForEach(sys->source_chain,
+                             SubSourceDelProxyCallbacks, sys->vout);
+    filter_chain_Delete(sys->source_chain);
+    free(sys->source_chain_current);
+    if (sys->vout)
+        filter_chain_ForEach(sys->filter_chain,
+                             SubFilterDelProxyCallbacks, sys->vout);
+    filter_chain_Delete(sys->filter_chain);
+    free(sys->filter_chain_current);
+    vlc_mutex_destroy(&sys->filter_chain_lock);
+    free(sys->source_chain_update);
+    free(sys->filter_chain_update);
+
+    /* Destroy all remaining subpictures */
+    for (size_t i = 0; i < sys->channels.size; ++i)
+        spu_channel_Clean(sys, &sys->channels.data[i]);
+
+    vlc_vector_destroy(&sys->channels);
+
+    vlc_mutex_destroy(&sys->lock);
+
+    vlc_mutex_destroy(&sys->prerender_lock);
+    vlc_cond_destroy(&sys->prerender_cond);
+    vlc_cond_destroy(&sys->prerender_output_cond);
+    vlc_vector_clear(&sys->prerender_vector);
+    video_format_Clean(&sys->prerender_fmtdst);
+    video_format_Clean(&sys->prerender_fmtsrc);
+}
+
+/**
+ * Destroy the subpicture unit
+ *
+ * \param p_this the parent object which destroys the subpicture unit
+ */
+void spu_Destroy(spu_t *spu)
+{
+    spu_private_t *sys = spu->p;
+    /* stop prerendering */
+    vlc_cancel(sys->prerender);
+    vlc_join(sys->prerender, NULL);
+    /* delete filters and free resources */
+    spu_Cleanup(spu);
+    vlc_object_delete(spu);
+}
+
 #undef spu_Create
 /**
  * Creates the subpicture unit
@@ -1447,51 +1669,22 @@ spu_t *spu_Create(vlc_object_t *object, vout_thread_t *vout)
     sys->last_sort_date = -1;
     sys->vout = vout;
 
-    return spu;
-}
-
-/**
- * Destroy the subpicture unit
- *
- * \param p_this the parent object which destroys the subpicture unit
- */
-void spu_Destroy(spu_t *spu)
-{
-    spu_private_t *sys = spu->p;
-
-    if (sys->text)
-        FilterRelease(sys->text);
-
-    if (sys->scale_yuvp)
-        FilterRelease(sys->scale_yuvp);
-
-    if (sys->scale)
-        FilterRelease(sys->scale);
-
-    filter_chain_ForEach(sys->source_chain, SubSourceClean, spu);
-    if (sys->vout)
-        filter_chain_ForEach(sys->source_chain,
-                             SubSourceDelProxyCallbacks, sys->vout);
-    filter_chain_Delete(sys->source_chain);
-    free(sys->source_chain_current);
-    if (sys->vout)
-        filter_chain_ForEach(sys->filter_chain,
-                             SubFilterDelProxyCallbacks, sys->vout);
-    filter_chain_Delete(sys->filter_chain);
-    free(sys->filter_chain_current);
-    vlc_mutex_destroy(&sys->filter_chain_lock);
-    free(sys->source_chain_update);
-    free(sys->filter_chain_update);
-
-    /* Destroy all remaining subpictures */
-    for (size_t i = 0; i < sys->channels.size; ++i)
-        spu_channel_Clean(&sys->channels.data[i]);
-
-    vlc_vector_destroy(&sys->channels);
-
-    vlc_mutex_destroy(&sys->lock);
+    vlc_mutex_init(&sys->prerender_lock);
+    vlc_cond_init(&sys->prerender_cond);
+    vlc_cond_init(&sys->prerender_output_cond);
+    vlc_vector_init(&sys->prerender_vector);
+    video_format_Init(&sys->prerender_fmtdst, 0);
+    video_format_Init(&sys->prerender_fmtsrc, 0);
+    sys->prerender_chroma_list = NULL;
+    sys->p_processed = NULL;
+    if(vlc_clone(&sys->prerender, spu_PrerenderThread, spu, VLC_THREAD_PRIORITY_VIDEO))
+    {
+        spu_Cleanup(spu);
+        vlc_object_delete(spu);
+        spu = NULL;
+    }
 
-    vlc_object_delete(spu);
+    return spu;
 }
 
 /**
@@ -1652,9 +1845,27 @@ void spu_PutSubpicture(spu_t *spu, subpicture_t *subpic)
         subpicture_Delete(subpic);
         return;
     }
+    spu_PrerenderEnqueue(sys, subpic);
     vlc_mutex_unlock(&sys->lock);
 }
 
+static const vlc_fourcc_t chroma_list_default_yuv[] = {
+    VLC_CODEC_YUVA,
+    VLC_CODEC_RGBA,
+    VLC_CODEC_ARGB,
+    VLC_CODEC_BGRA,
+    VLC_CODEC_YUVP,
+    0,
+};
+static const vlc_fourcc_t chroma_list_default_rgb[] = {
+    VLC_CODEC_RGBA,
+    VLC_CODEC_ARGB,
+    VLC_CODEC_BGRA,
+    VLC_CODEC_YUVA,
+    VLC_CODEC_YUVP,
+    0,
+};
+
 subpicture_t *spu_Render(spu_t *spu,
                          const vlc_fourcc_t *chroma_list,
                          const video_format_t *fmt_dst,
@@ -1691,27 +1902,13 @@ subpicture_t *spu_Render(spu_t *spu,
     /* Run subpicture sources */
     filter_chain_SubSource(sys->source_chain, spu, system_now);
 
-    static const vlc_fourcc_t chroma_list_default_yuv[] = {
-        VLC_CODEC_YUVA,
-        VLC_CODEC_RGBA,
-        VLC_CODEC_ARGB,
-        VLC_CODEC_BGRA,
-        VLC_CODEC_YUVP,
-        0,
-    };
-    static const vlc_fourcc_t chroma_list_default_rgb[] = {
-        VLC_CODEC_RGBA,
-        VLC_CODEC_ARGB,
-        VLC_CODEC_BGRA,
-        VLC_CODEC_YUVA,
-        VLC_CODEC_YUVP,
-        0,
-    };
-
     if (!chroma_list || *chroma_list == 0)
         chroma_list = vlc_fourcc_IsYUV(fmt_dst->i_chroma) ? chroma_list_default_yuv
                                                           : chroma_list_default_rgb;
 
+    /* wake up prerenderer, we have some video size and chroma */
+    spu_PrerenderWake(sys, fmt_dst, fmt_src, chroma_list);
+
     vlc_mutex_lock(&sys->lock);
 
     size_t subpicture_count;
@@ -1730,6 +1927,9 @@ subpicture_t *spu_Render(spu_t *spu,
     for (size_t i = 0; i < subpicture_count; i++) {
         spu_render_entry_t *entry = &subpicture_array[i];
         subpicture_t *subpic = entry->subpic;
+
+        spu_PrerenderSync(sys, entry->subpic);
+
         if (!subpic->updater.pf_validate)
             continue;
 
@@ -1789,10 +1989,14 @@ ssize_t spu_RegisterChannel(spu_t *spu)
     return spu_RegisterChannelInternal(spu, NULL, NULL);
 }
 
-static void spu_channel_Clear(struct spu_channel *channel)
+static void spu_channel_Clear(spu_private_t *sys,
+                              struct spu_channel *channel)
 {
     for (size_t i = 0; i < channel->entries.size; i++)
+    {
+        spu_PrerenderCancel(sys, channel->entries.data[i].subpic);
         spu_channel_DeleteAt(channel, i);
+    }
 }
 
 void spu_ClearChannel(spu_t *spu, size_t channel_id)
@@ -1800,7 +2004,7 @@ void spu_ClearChannel(spu_t *spu, size_t channel_id)
     spu_private_t *sys = spu->p;
     vlc_mutex_lock(&sys->lock);
     struct spu_channel *channel = spu_GetChannel(spu, channel_id);
-    spu_channel_Clear(channel);
+    spu_channel_Clear(sys, channel);
     if (channel->clock)
     {
         vlc_clock_Reset(channel->clock);
@@ -1815,7 +2019,7 @@ void spu_UnregisterChannel(spu_t *spu, size_t channel_id)
 
     vlc_mutex_lock(&sys->lock);
     struct spu_channel *channel = spu_GetChannel(spu, channel_id);
-    spu_channel_Clean(channel);
+    spu_channel_Clean(sys, channel);
     vlc_vector_remove(&sys->channels, channel_id);
     vlc_mutex_unlock(&sys->lock);
 }
-- 
2.20.1



More information about the vlc-devel mailing list