[vlc-devel] [PATCH v3] codec/audiotoolbox_midi: Add AudioToolbox MIDI decoder
Thomas Guillem
thomas at gllm.fr
Wed Nov 29 16:34:24 CET 2017
On Wed, Nov 29, 2017, at 16:22, Marvin Scholz wrote:
> ---
> NEWS | 2 +
> configure.ac | 6 +
> modules/MODULES_LIST | 3 +-
> modules/codec/Makefile.am | 6 +
> modules/codec/audiotoolbox_midi.c | 403
> ++++++++++++++++++++++++++++++++++++++
> po/POTFILES.in | 1 +
> 6 files changed, 420 insertions(+), 1 deletion(-)
> create mode 100644 modules/codec/audiotoolbox_midi.c
>
> diff --git a/NEWS b/NEWS
> index ddd1626f37..edc5d61d37 100644
> --- a/NEWS
> +++ b/NEWS
> @@ -99,6 +99,8 @@ Decoder:
> * Extend MicroDVD support with color, fontname, size, position
> extensions
> * BluRay text subtitles are now decoded
> * Improved Closed Captions detection and optional CEA-708 decoder
> + * New MIDI decoder for macOS and iOS using the AudioToolbox framework,
> works
> + without a soundfont or with SoundFont2 and DLS soundfonts
>
> Demuxers:
> * Support HD-DVD .evo (H.264, VC-1, MPEG-2, PCM, AC-3, E-AC3, MLP, DTS)
> diff --git a/configure.ac b/configure.ac
> index 62ce59aff5..fc4e05f9ed 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -3894,6 +3894,12 @@ AC_CHECK_HEADERS(VideoToolbox/VideoToolbox.h, [
> ])
>
> dnl
> +dnl AudioToolbox MIDI plugin
> +AC_CHECK_HEADERS([AudioToolbox/AudioToolbox.h], [
> + VLC_ADD_PLUGIN([audiotoolboxmidi])
> +])
> +
> +dnl
> dnl ncurses module
> dnl
> AC_ARG_ENABLE(ncurses,
> diff --git a/modules/MODULES_LIST b/modules/MODULES_LIST
> index 014c5ddafb..146e537a56 100644
> --- a/modules/MODULES_LIST
> +++ b/modules/MODULES_LIST
> @@ -1,4 +1,4 @@
> -List of vlc plugins (490):
> +List of vlc plugins (491):
> $Id$
>
> * a52: A/52 audio decoder plugin, using liba52
> @@ -53,6 +53,7 @@ $Id$
> * audiobargraph_a: audiobargraph audio plugin
> * audiobargraph_v: audiobargraph video plugin
> * audioscrobbler: AudioScrobbler/Last.fm submission plugin
> + * audiotoolbox_midi: AudioToolbox MIDI decoder plugin for macOS
> * audiounit_ios: AudioUnit output plugin for iOS
> * auhal: Audio output for Mac OS X based on the AUHAL API
> * avahi: Zeroconf services discovery
> diff --git a/modules/codec/Makefile.am b/modules/codec/Makefile.am
> index 075ee2a0e9..6a8a72019c 100644
> --- a/modules/codec/Makefile.am
> +++ b/modules/codec/Makefile.am
> @@ -60,6 +60,12 @@ if HAVE_DARWIN
> libfluidsynth_plugin_la_LDFLAGS +=
> -Wl,-framework,CoreFoundation,-framework,CoreServices
> endif
>
> +libaudiotoolboxmidi_plugin_la_SOURCES = codec/audiotoolbox_midi.c
> +libaudiotoolboxmidi_plugin_la_LDFLAGS = $(AM_LDFLAGS) -rpath
> '$(codecdir)'
> +libaudiotoolboxmidi_plugin_la_LDFLAGS +=
> -Wl,-framework,CoreFoundation,-framework,AudioUnit,-framework,AudioToolbox
> +EXTRA_LTLIBRARIES += libaudiotoolboxmidi_plugin.la
> +codec_LTLIBRARIES += $(LTLIBaudiotoolboxmidi)
> +
> liblpcm_plugin_la_SOURCES = codec/lpcm.c
> codec_LTLIBRARIES += liblpcm_plugin.la
>
> diff --git a/modules/codec/audiotoolbox_midi.c
> b/modules/codec/audiotoolbox_midi.c
> new file mode 100644
> index 0000000000..229df38328
> --- /dev/null
> +++ b/modules/codec/audiotoolbox_midi.c
> @@ -0,0 +1,403 @@
> +/*****************************************************************************
> + * audiotoolbox_midi.c: Software MIDI synthesizer using AudioToolbox
> +
> *****************************************************************************
> + * Copyright (C) 2017 VLC authors and VideoLAN
> + * $Id$
> + *
> + * Authors: Marvin Scholz <epirat07 at gmail dot com>
> + *
> + * Based on the fluidsynth module by Rémi Denis-Courmont
> + *
> + * This program is free software; you can redistribute it and/or modify
> it
> + * under the terms of the GNU Lesser General Public License as published
> by
> + * the Free Software Foundation; either version 2.1 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> License
> + * along with this program; if not, write to the Free Software
> Foundation,
> + * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
> +
> *****************************************************************************/
> +
> +#ifdef HAVE_CONFIG_H
> +# include "config.h"
> +#endif
> +
> +#include <vlc_common.h>
> +#include <vlc_plugin.h>
> +#include <vlc_codec.h>
> +#include <vlc_dialog.h>
> +
> +#include <CoreFoundation/CoreFoundation.h>
> +#include <AudioUnit/AudioUnit.h>
> +#include <AudioToolbox/AudioToolbox.h>
> +
> +#include <TargetConditionals.h>
> +
> +#ifndef on_err_goto
> +#define on_err_goto(errorCode, exceptionLabel) \
> +do { if ((errorCode) != noErr) goto exceptionLabel; \
> +} while ( 0 )
> +#endif
> +
> +#define SOUNDFONT_TEXT N_("Sound font file")
> +#define SOUNDFONT_LONGTEXT N_( \
> + "The Sound Font file (SF2/DLS) to use for synthesis.")
> +
> +static int Open (vlc_object_t *);
> +static void Close (vlc_object_t *);
> +
> +#define CFG_PREFIX "aumidi-"
> +
> +vlc_module_begin()
> + set_description(N_("AudioToolbox MIDI synthesizer"))
> + set_capability("audio decoder", 100)
> + set_shortname(N_("AUMIDI"))
> + set_category(CAT_INPUT)
> + set_subcategory(SUBCAT_INPUT_ACODEC)
> + set_callbacks(Open, Close)
> + add_loadfile(CFG_PREFIX "soundfont", "",
> + SOUNDFONT_TEXT, SOUNDFONT_LONGTEXT, false)
> +vlc_module_end()
> +
> +
> +struct decoder_sys_t
> +{
> + AUGraph graph;
> + AudioUnit synthUnit;
> + AudioUnit outputUnit;
> + date_t end_date;
> +};
> +
> +static int DecodeBlock (decoder_t *p_dec, block_t *p_block);
> +static void Flush (decoder_t *);
> +
> +/* MIDI constants */
> +enum
> +{
> + kMidiMessage_NoteOff = 0x80,
> + kMidiMessage_NoteOn = 0x90,
> + kMidiMessage_PolyPressure = 0xA0,
> + kMidiMessage_ControlChange = 0xB0,
> + kMidiMessage_ProgramChange = 0xC0,
> + kMidiMessage_ChannelPressure = 0xD0,
> + kMidiMessage_PitchWheel = 0xE0,
> + kMidiMessage_SysEx = 0xF0,
> +
> + kMidiMessage_BankMSBControl = 0,
> + kMidiMessage_BankLSBControl = 32,
> +
> + /* Values for kMidiMessage_ControlChange */
> + kMidiController_AllSoundOff = 0x78,
> + kMidiController_ResetAllControllers = 0x79,
> + kMidiController_AllNotesOff = 0x7B
> +};
> +
> +/* Helper functions */
> +static OSStatus AddAppleAUNode(AUGraph graph, OSType type, OSType
> subtype, AUNode *node)
> +{
> + AudioComponentDescription cDesc = {};
> + cDesc.componentType = type;
> + cDesc.componentSubType = subtype;
> + cDesc.componentManufacturer = kAudioUnitManufacturer_Apple;
> +
> + return AUGraphAddNode(graph, &cDesc, node);
> +}
> +
> +static OSStatus CreateAUGraph(AUGraph *outGraph, AudioUnit *outSynth,
> AudioUnit *outOut)
> +{
> + OSStatus res;
> +
> + // AudioUnit nodes
> + AUNode synthNode, limiterNode, outNode;
> +
> + // Create the Graph to which we will add our nodes
> + on_err_goto(res = NewAUGraph(outGraph), bailout);
> +
> + // Create/add the MIDI synthesizer node (DLS Synth)
> +#if TARGET_OS_IPHONE
> + // On iOS/tvOS use MIDISynth, DLSSynth does not exist there
> + on_err_goto(res = AddAppleAUNode(*outGraph,
> + kAudioUnitType_MusicDevice,
> + kAudioUnitSubType_MIDISynth,
> + &synthNode), bailout);
> +#else
> + // Prefer DLSSynth on macOS, as it has a better default behavior
> + on_err_goto(res = AddAppleAUNode(*outGraph,
> + kAudioUnitType_MusicDevice,
> + kAudioUnitSubType_DLSSynth,
> + &synthNode), bailout);
> +#endif
> +
> + // Create/add the peak limiter node
> + on_err_goto(res = AddAppleAUNode(*outGraph,
> + kAudioUnitType_Effect,
> + kAudioUnitSubType_PeakLimiter,
> + &limiterNode), bailout);
> +
> + // Create/add the output node (GenericOutput)
> + on_err_goto(res = AddAppleAUNode(*outGraph,
> + kAudioUnitType_Output,
> + kAudioUnitSubType_GenericOutput,
> + &outNode), bailout);
> +
> + // Open the Graph, this opens the units that belong to the graph
> + // so that we can connect them
> + on_err_goto(res = AUGraphOpen(*outGraph), bailout);
> +
> + // Connect the synthesizer node to the limiter
> + on_err_goto(res = AUGraphConnectNodeInput(*outGraph, synthNode, 0,
> limiterNode, 0), bailout);
> + // Connect the limiter node to the output
> + on_err_goto(res = AUGraphConnectNodeInput(*outGraph, limiterNode, 0,
> outNode, 0), bailout);
> +
> + // Get reference to the synthesizer node
> + on_err_goto(res = AUGraphNodeInfo(*outGraph, synthNode, 0,
> outSynth), bailout);
> + // Get reference to the output node
> + on_err_goto(res = AUGraphNodeInfo(*outGraph, outNode, 0, outOut),
> bailout);
> +
> +bailout:
> + return res;
> +}
> +
> +static int SetSoundfont(decoder_t *p_dec, AudioUnit synthUnit, const
> char *sfPath) {
> + if (!sfPath) {
> + msg_Dbg(p_dec, "using default soundfont");
> + return VLC_SUCCESS;
> + }
> +
> + msg_Dbg(p_dec, "using custom soundfont: '%s'", sfPath);
> + CFURLRef url =
> CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault,
> + (const UInt8
> *)sfPath,
> +
> strlen(sfPath), false);
> + if (unlikely(url == NULL))
> + return VLC_ENOMEM;
> +
> + OSStatus status = AudioUnitSetProperty(synthUnit,
> +
> kMusicDeviceProperty_SoundBankURL,
> + kAudioUnitScope_Global, 0,
> + &url, sizeof(url));
> + CFRelease(url);
> +
> + if (status != noErr) {
> + msg_Err(p_dec, "failed setting custom SoundFont for MIDI
> synthesis (%i)", status);
> + return VLC_EGENERIC;
> + }
> + return VLC_SUCCESS;
> +}
> +
> +static int Open(vlc_object_t *p_this)
> +{
> + decoder_t *p_dec = (decoder_t *)p_this;
> + OSStatus status = noErr;
> +
> + if (p_dec->fmt_in.i_codec != VLC_CODEC_MIDI)
> + return VLC_EGENERIC;
> +
> + decoder_sys_t *p_sys = malloc(sizeof (*p_sys));
> + if (unlikely(p_sys == NULL))
> + return VLC_ENOMEM;
> +
> + status = CreateAUGraph(&p_sys->graph, &p_sys->synthUnit,
> &p_sys->outputUnit);
> + if (unlikely(status != noErr)) {
> + msg_Err(p_dec, "failed to create audiograph (%i)", status);
> + return VLC_EGENERIC;
> + }
> +
> + // Set custom soundfont
> + char *sfPath = var_InheritString(p_dec, CFG_PREFIX "soundfont");
> + int ret = SetSoundfont(p_dec, p_sys->synthUnit, sfPath);
> + free(sfPath);
> + if (unlikely(ret != VLC_SUCCESS))
> + return ret;
> +
> + // Prepare AudioUnit output audio format info
> + AudioStreamBasicDescription ASBD = {};
> + unsigned bytesPerSample = sizeof(Float32);
> + ASBD.mFormatID = kAudioFormatLinearPCM;
> + ASBD.mFormatFlags = kAudioFormatFlagIsFloat |
> kAudioFormatFlagIsPacked;
> + ASBD.mSampleRate = 44100;
> + ASBD.mFramesPerPacket = 1;
> + ASBD.mChannelsPerFrame = 2;
> + ASBD.mBytesPerFrame = bytesPerSample * ASBD.mChannelsPerFrame;
> + ASBD.mBytesPerPacket = ASBD.mBytesPerFrame * ASBD.mFramesPerPacket;
> + ASBD.mBitsPerChannel = 8 * bytesPerSample;
> +
> + // Set AudioUnit format
> + status = AudioUnitSetProperty(p_sys->outputUnit,
> + kAudioUnitProperty_StreamFormat,
> + kAudioUnitScope_Output, 0, &ASBD,
> + sizeof(AudioStreamBasicDescription));
> + if (unlikely(status != noErr)) {
> + msg_Err(p_dec, "failed setting output format for output unit
> (%i)", status);
> + return VLC_EGENERIC;
> + }
> +
> + // Prepare the AU
> + status = AUGraphInitialize (p_sys->graph);
> + if (unlikely(status != noErr)) {
> + if (status == kAudioUnitErr_InvalidFile)
> + msg_Err(p_dec, "failed initializing audiograph: invalid
> soundfont file");
> + else
> + msg_Err(p_dec, "failed initializing audiograph (%i)",
> status);
> + return VLC_EGENERIC;
> + }
> +
> + // Prepare MIDI soundbank
> + MusicDeviceMIDIEvent(p_sys->synthUnit,
> + kMidiMessage_ControlChange,
> + kMidiMessage_BankMSBControl, 0, 0);
> +
> + // Start the AU
> + status = AUGraphStart(p_sys->graph);
> + if (unlikely(status != noErr)) {
> + msg_Err(p_dec, "failed starting audiograph (%i)", status);
> + return VLC_EGENERIC;
> + }
> +
> + // Set VLC output audio format info
> + p_dec->fmt_out.i_codec = VLC_CODEC_FL32;
> + p_dec->fmt_out.audio.i_bitspersample = 32;
> + p_dec->fmt_out.audio.i_rate = 44100;
> + p_dec->fmt_out.audio.i_channels = 2;
> + p_dec->fmt_out.audio.i_physical_channels = AOUT_CHAN_LEFT |
> AOUT_CHAN_RIGHT;
> +
> + // Initialize date (for PTS)
> + date_Init(&p_sys->end_date, p_dec->fmt_out.audio.i_rate, 1);
> + date_Set(&p_sys->end_date, 0);
> +
> + p_dec->p_sys = p_sys;
> + p_dec->pf_decode = DecodeBlock;
> + p_dec->pf_flush = Flush;
> + return VLC_SUCCESS;
> +}
> +
> +
> +static void Close (vlc_object_t *p_this)
> +{
> + decoder_sys_t *p_sys = ((decoder_t *)p_this)->p_sys;
> +
> + if (p_sys->graph) {
> + AUGraphStop(p_sys->graph);
> + DisposeAUGraph(p_sys->graph);
> + }
> + free(p_sys);
> +}
> +
> +static void Flush (decoder_t *p_dec)
> +{
> + decoder_sys_t *p_sys = p_dec->p_sys;
> +
> + date_Set(&p_sys->end_date, VLC_TS_INVALID);
> +
> + // Turn all sound on all channels off
> + // else 'old' notes could still be playing
> + for (unsigned channel = 0; channel < 16; channel++) {
> + MusicDeviceMIDIEvent(p_sys->synthUnit,
> kMidiMessage_ControlChange | channel, kMidiController_AllSoundOff, 0, 0);
> + }
> +}
> +
> +static int DecodeBlock (decoder_t *p_dec, block_t *p_block)
> +{
> + decoder_sys_t *p_sys = p_dec->p_sys;
> + block_t *p_out = NULL;
> + OSStatus status = noErr;
> +
> + if (p_block == NULL) /* No Drain */
> + return VLCDEC_SUCCESS;
> +
> + if (p_block->i_flags &
> (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
> + Flush(p_dec);
> + if (p_block->i_flags & BLOCK_FLAG_CORRUPTED) {
> + block_Release(p_block);
> + return VLCDEC_SUCCESS;
> + }
> + }
> +
> + if (p_block->i_pts > VLC_TS_INVALID && !date_Get(&p_sys->end_date))
> {
> + date_Set(&p_sys->end_date, p_block->i_pts);
> + } else if (p_block->i_pts < date_Get(&p_sys->end_date)) {
> + msg_Warn(p_dec, "MIDI message in the past?");
> + goto drop;
> + }
> +
> + if (p_block->i_buffer < 1)
> + goto drop;
> +
> + uint8_t event = p_block->p_buffer[0];
> + uint8_t data1 = (p_block->i_buffer > 1) ? (p_block->p_buffer[1]) :
> 0;
> + uint8_t data2 = (p_block->i_buffer > 2) ? (p_block->p_buffer[2]) :
> 0;
> +
> + switch (event & 0xF0)
> + {
> + case kMidiMessage_NoteOff:
> + case kMidiMessage_NoteOn:
> + case kMidiMessage_PolyPressure:
> + case kMidiMessage_ControlChange:
> + case kMidiMessage_ProgramChange:
> + case kMidiMessage_ChannelPressure:
> + case kMidiMessage_PitchWheel:
> + MusicDeviceMIDIEvent(p_sys->synthUnit, event, data1, data2,
> 0);
> + break;
> +
> + case kMidiMessage_SysEx:
> + // SysEx not handled
> + break;
> +
> + default:
> + msg_Warn(p_dec, "unhandled MIDI event: %x", event & 0xF0);
> + break;
> + }
> +
> + // Calculate frame count
> + // Simplification of 44100 / 1000000
> + // TODO: Other samplerates
> + unsigned frames =
> + (p_block->i_pts - date_Get(&p_sys->end_date)) * 441 / 10000;
> +
> + if (frames == 0)
> + goto drop;
> +
> + if (decoder_UpdateAudioFormat(p_dec))
> + goto drop;
What about my first comment?
Ok I wrote a typo: here is my fixed comment again:
If dec->fmt_out *can not* be changed while decoding, you should put this
code
in the Open() function just after setting fmt_out.
> +
> + p_out = decoder_NewAudioBuffer(p_dec, frames);
> + if (p_out == NULL)
> + goto drop;
> +
> + p_out->i_pts = date_Get(&p_sys->end_date );
> + p_out->i_length = date_Increment(&p_sys->end_date, frames)
> + - p_out->i_pts;
> +
> + // Prepare Timestamp for the AudioUnit render call
> + AudioTimeStamp timestamp = {};
> + timestamp.mFlags = kAudioTimeStampWordClockTimeValid;
> + timestamp.mWordClockTime = p_out->i_pts;
> +
> + // Prepare Buffer for the AudioUnit render call
> + AudioBufferList bufferList;
> + bufferList.mNumberBuffers = 1;
> + bufferList.mBuffers[0].mNumberChannels = 2;
> + bufferList.mBuffers[0].mDataByteSize = frames * sizeof(Float32) * 2;
> + bufferList.mBuffers[0].mData = p_out->p_buffer;
> +
> + status = AudioUnitRender(p_sys->outputUnit,
> + NULL,
> + ×tamp, 0,
> + frames, &bufferList);
> +
> + if (status != noErr) {
> + msg_Warn(p_dec, "rendering audio unit failed: %i", status);
> + block_Release(p_out);
> + p_out = NULL;
> + }
> +
> +drop:
> + block_Release(p_block);
> + if (p_out != NULL)
> + decoder_QueueAudio(p_dec, p_out);
> + return VLCDEC_SUCCESS;
> +}
> diff --git a/po/POTFILES.in b/po/POTFILES.in
> index 040e65cb10..c1a1facd88 100644
> --- a/po/POTFILES.in
> +++ b/po/POTFILES.in
> @@ -292,6 +292,7 @@ modules/codec/adpcm.c
> modules/codec/aes3.c
> modules/codec/aom.c
> modules/codec/araw.c
> +modules/codec/audiotoolbox_midi.c
> modules/codec/arib/aribsub.c
> modules/codec/avcodec/avcodec.c
> modules/codec/avcodec/avcodec.h
> --
> 2.13.6 (Apple Git-96)
>
> _______________________________________________
> vlc-devel mailing list
> To unsubscribe or modify your subscription options:
> https://mailman.videolan.org/listinfo/vlc-devel
More information about the vlc-devel
mailing list