[vlc-devel] [PATCH 2/2] audio_filter: add RNNoise denoiser filter
Steve Lhomme
robux4 at ycbcr.xyz
Thu Oct 15 07:42:53 CEST 2020
LGTM apart nitpicking below.
On 2020-10-15 6:00, Tristan Matthews wrote:
> ---
> NEWS | 3 +
> configure.ac | 5 +
> modules/audio_filter/Makefile.am | 8 ++
> modules/audio_filter/rnnoise.c | 179 +++++++++++++++++++++++++++++++
> po/POTFILES.in | 1 +
> 5 files changed, 196 insertions(+)
> create mode 100644 modules/audio_filter/rnnoise.c
>
> diff --git a/NEWS b/NEWS
> index 50b8e7a9f6..f6e1a65c50 100644
> --- a/NEWS
> +++ b/NEWS
> @@ -81,6 +81,9 @@ Video output:
> * Remove Real demuxer plugin
> * Fix washed out black on NVIDIA cards with Direct3D9
>
> +Audio filter:
> + * Add RNNoise recurrent neural network denoiser
> +
> Video filter:
> * Update yadif
> * Remove remote OSD plugin
> diff --git a/configure.ac b/configure.ac
> index 55e4d17d86..4a19fdd999 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -3011,6 +3011,11 @@ AS_IF([test "${enable_speex}" != "no"], [
> ])
> AM_CONDITIONAL([HAVE_SPEEXDSP], [test "$have_speexdsp" = "yes"])
>
> +dnl
> +dnl RNNoise plugin - Recurrent neural network for audio noise reduction
> +dnl
> +PKG_ENABLE_MODULES_VLC([RNNOISE], [], [rnnoise], [Rnnoise denoiser], [auto])
> +
> dnl
> dnl Opus plugin
> dnl
> diff --git a/modules/audio_filter/Makefile.am b/modules/audio_filter/Makefile.am
> index 028b188bfe..c77a90cb15 100644
> --- a/modules/audio_filter/Makefile.am
> +++ b/modules/audio_filter/Makefile.am
> @@ -147,3 +147,11 @@ libspeex_resampler_plugin_la_LIBADD = $(SPEEXDSP_LIBS)
> if HAVE_SPEEXDSP
> audio_filter_LTLIBRARIES += libspeex_resampler_plugin.la
> endif
> +
> +librnnoise_plugin_la_SOURCES = audio_filter/rnnoise.c
> +librnnoise_plugin_la_CFLAGS = $(AM_CFLAGS) $(RNNOISE_CFLAGS)
> +librnnoise_plugin_la_LIBADD = $(RNNOISE_LIBS) $(LIBM)
> +librnnoise_plugin_la_LDFLAGS = $(AM_LDFLAGS) -rpath '$(audio_filterdir)'
> +
> +audio_filter_LTLIBRARIES += $(LTLIBrnnoise)
> +EXTRA_LTLIBRARIES += librnnoise_plugin.la
> diff --git a/modules/audio_filter/rnnoise.c b/modules/audio_filter/rnnoise.c
> new file mode 100644
> index 0000000000..1995c4949a
> --- /dev/null
> +++ b/modules/audio_filter/rnnoise.c
> @@ -0,0 +1,179 @@
> +/*****************************************************************************
> + * rnnoise.c : Recurrent neural network for audio noise reduction
> + *****************************************************************************
> + * Copyright © 2019 Tristan Matthews
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU Lesser General Public License as published by
> + * the Free Software Foundation; either version 2.1 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public License
> + * along with this program; if not, write to the Free Software Foundation,
> + * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
> + *****************************************************************************/
> +
> +#ifdef HAVE_CONFIG_H
> +# include "config.h"
> +#endif
> +
> +#include <vlc_common.h>
> +#include <vlc_aout.h>
> +#include <vlc_filter.h>
> +#include <vlc_plugin.h>
> +
> +#include <rnnoise.h>
> +
> +typedef struct
> +{
> + DenoiseState **p_sts;
> + bool b_first;
> +} filter_sys_t;
> +
> +static void
> +Flush(filter_t *p_filter)
> +{
> + filter_sys_t *p_sys = p_filter->p_sys;
> +
> + if (p_sys->p_sts) {
> + int i_channels = p_filter->fmt_in.audio.i_channels;
> + for (int i = 0; i < i_channels; i++) {
> + rnnoise_destroy( p_sys->p_sts[i] );
> + }
> + free(p_sys->p_sts);
> + p_sys->p_sts = NULL;
> + p_sys->b_first = true;
> + }
> +}
> +
> +static DenoiseState **init_denoise_state(unsigned i_channels)
> +{
> + DenoiseState **p_sts = malloc(i_channels * sizeof(DenoiseState *));
> + if (unlikely(!p_sts))
> + return NULL;
> +
> + for (unsigned i = 0; i < i_channels; i++)
> + {
> + p_sts[i] = rnnoise_create(NULL);
> + if (unlikely(!p_sts[i]))
> + {
> + for (unsigned j = 0; j < i; j++)
> + rnnoise_destroy( p_sts[j] );
> +
> + return NULL;
> + }
> + }
> +
> + return p_sts;
> +}
> +
> +
> +#define FRAME_SIZE 480
> +
> +static block_t *Process(filter_t *p_filter, block_t *p_block)
> +{
> + filter_sys_t *p_sys = p_filter->p_sys;
> + float *p_buffer = (float *)p_block->p_buffer;
> + const int i_channels = p_filter->fmt_in.audio.i_channels;
> +
> + float *tmp = malloc(FRAME_SIZE * i_channels * sizeof(*tmp));
> + if (unlikely(!tmp))
> + {
> + return p_block;
> + }
> +
> + if (unlikely(p_sys->p_sts == NULL))
> + {
> + /* Can happen after a flush */
> + p_sys->p_sts = init_denoise_state(i_channels);
> + if (p_sys->p_sts == NULL)
> + return p_block;
> + }
> +
> + for (int i_nb_samples = p_block->i_nb_samples; i_nb_samples > 0; i_nb_samples -= FRAME_SIZE)
> + {
> + /* handle case where we have fewer than FRAME_SIZE samples left to process */
> + const unsigned frame_size = __MIN(FRAME_SIZE, i_nb_samples);
> +
> + /* rnnoise processes blocks of 480 samples, and expects input to be in the 32768 scale. */
> + for (unsigned i = 0; i < frame_size; i++) {
> + for (int j = 0; j < i_channels; j++) {
> + tmp[i + j * frame_size] = p_buffer[i * i_channels + j] * 32768.f;
> + }
> + }
> +
> + for (int i = 0; i < i_channels; i++) {
> + rnnoise_process_frame(p_sys->p_sts[i], tmp + i * frame_size , tmp + i * frame_size);
> + }
> +
> + /* Skip writing first frame to output (as per the examples) I guess to prime rnnoise? */
> + if(!p_sys->b_first)
> + {
> + for (unsigned i = 0; i < frame_size; i++) {
> + for (int j = 0; j < i_channels; j++) {
> + p_buffer[i * i_channels + j] = tmp[i + j * frame_size] / 32768.f;
> + }
> + }
> + }
> + else
> + {
> + p_sys->b_first = false;
> + }
> + p_buffer += frame_size * i_channels;
> + }
> +
> + free(tmp);
> + return p_block;
> +}
> +
> +static void Close( filter_t *obj )
> +{
> + filter_t *p_filter = (filter_t *)obj;
> + Flush( p_filter );
> + free( p_filter->p_sys );
> +}
> +
> +static int Open (vlc_object_t *obj)
> +{
> + filter_t *p_filter = (filter_t *)obj;
> +
> + filter_sys_t *p_sys = p_filter->p_sys = malloc(sizeof(filter_sys_t));
You can allocate the p_sys with vlc_obj_malloc() so you don't have to
free it manually in Close. (and then it only calls Flush, surprise!)
> + if(unlikely(!p_sys))
> + return VLC_ENOMEM;
> +
> + p_sys->b_first = true;
> +
> + int i_channels = p_filter->fmt_in.audio.i_channels;
> + p_sys->p_sts = init_denoise_state(i_channels);
> + if (unlikely(!p_sys->p_sts))
> + {
> + free( p_sys );
Which becomes vlc_obj_free().
> + return VLC_ENOMEM;
> + }
> +
> + p_filter->fmt_in.audio.i_format = VLC_CODEC_FL32;
> + aout_FormatPrepare(&p_filter->fmt_in.audio);
> + p_filter->fmt_out.audio = p_filter->fmt_in.audio;
> +
> + static const struct vlc_filter_operations filter_ops =
> + {
> + .filter_audio = Process, .flush = Flush, .close = Close,
> + };
> + p_filter->ops = &filter_ops;
> +
> + return VLC_SUCCESS;
> +}
> +
> +vlc_module_begin ()
> + set_shortname (N_("RNNoise"))
> + set_description (N_("RNNoise filter"))
> + set_category (CAT_AUDIO)
> + set_subcategory (SUBCAT_AUDIO_AFILTER)
> + set_capability ("audio filter", 0)
> + set_callback( Open )
> +vlc_module_end ()
> diff --git a/po/POTFILES.in b/po/POTFILES.in
> index 2395e6fc0a..e924ec061c 100644
> --- a/po/POTFILES.in
> +++ b/po/POTFILES.in
> @@ -247,6 +247,7 @@ modules/audio_filter/resampler/soxr.c
> modules/audio_filter/resampler/speex.c
> modules/audio_filter/resampler/src.c
> modules/audio_filter/resampler/ugly.c
> +modules/audio_filter/rnnoise.c
> modules/audio_filter/scaletempo.c
> modules/audio_filter/spatializer/spatializer.cpp
> modules/audio_filter/stereo_widen.c
> --
> 2.25.1
>
> _______________________________________________
> vlc-devel mailing list
> To unsubscribe or modify your subscription options:
> https://mailman.videolan.org/listinfo/vlc-devel
>
More information about the vlc-devel
mailing list