[vlc-devel] [PATCH v3] codec/audiotoolbox_midi: Add AudioToolbox MIDI decoder

Marvin Scholz epirat07 at gmail.com
Wed Nov 29 16:22:06 CET 2017


---
 NEWS                              |   2 +
 configure.ac                      |   6 +
 modules/MODULES_LIST              |   3 +-
 modules/codec/Makefile.am         |   6 +
 modules/codec/audiotoolbox_midi.c | 403 ++++++++++++++++++++++++++++++++++++++
 po/POTFILES.in                    |   1 +
 6 files changed, 420 insertions(+), 1 deletion(-)
 create mode 100644 modules/codec/audiotoolbox_midi.c

diff --git a/NEWS b/NEWS
index ddd1626f37..edc5d61d37 100644
--- a/NEWS
+++ b/NEWS
@@ -99,6 +99,8 @@ Decoder:
  * Extend MicroDVD support with color, fontname, size, position extensions
  * BluRay text subtitles are now decoded
  * Improved Closed Captions detection and optional CEA-708 decoder
+ * New MIDI decoder for macOS and iOS using the AudioToolbox framework, works
+   without a soundfont or with SoundFont2 and DLS soundfonts
 
 Demuxers:
  * Support HD-DVD .evo (H.264, VC-1, MPEG-2, PCM, AC-3, E-AC3, MLP, DTS)
diff --git a/configure.ac b/configure.ac
index 62ce59aff5..fc4e05f9ed 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3894,6 +3894,12 @@ AC_CHECK_HEADERS(VideoToolbox/VideoToolbox.h, [
   ])
 
 dnl
+dnl  AudioToolbox MIDI plugin
+AC_CHECK_HEADERS([AudioToolbox/AudioToolbox.h], [
+  VLC_ADD_PLUGIN([audiotoolboxmidi])
+])
+
+dnl
 dnl  ncurses module
 dnl
 AC_ARG_ENABLE(ncurses,
diff --git a/modules/MODULES_LIST b/modules/MODULES_LIST
index 014c5ddafb..146e537a56 100644
--- a/modules/MODULES_LIST
+++ b/modules/MODULES_LIST
@@ -1,4 +1,4 @@
-List of vlc plugins (490):
+List of vlc plugins (491):
 $Id$
 
  * a52: A/52 audio decoder plugin, using liba52
@@ -53,6 +53,7 @@ $Id$
  * audiobargraph_a: audiobargraph audio plugin
  * audiobargraph_v: audiobargraph video plugin
  * audioscrobbler: AudioScrobbler/Last.fm submission plugin
+ * audiotoolbox_midi: AudioToolbox MIDI decoder plugin for macOS
  * audiounit_ios: AudioUnit output plugin for iOS
  * auhal: Audio output for Mac OS X based on the AUHAL API
  * avahi: Zeroconf services discovery
diff --git a/modules/codec/Makefile.am b/modules/codec/Makefile.am
index 075ee2a0e9..6a8a72019c 100644
--- a/modules/codec/Makefile.am
+++ b/modules/codec/Makefile.am
@@ -60,6 +60,12 @@ if HAVE_DARWIN
 libfluidsynth_plugin_la_LDFLAGS += -Wl,-framework,CoreFoundation,-framework,CoreServices
 endif
 
+libaudiotoolboxmidi_plugin_la_SOURCES = codec/audiotoolbox_midi.c
+libaudiotoolboxmidi_plugin_la_LDFLAGS = $(AM_LDFLAGS) -rpath '$(codecdir)'
+libaudiotoolboxmidi_plugin_la_LDFLAGS += -Wl,-framework,CoreFoundation,-framework,AudioUnit,-framework,AudioToolbox
+EXTRA_LTLIBRARIES += libaudiotoolboxmidi_plugin.la
+codec_LTLIBRARIES += $(LTLIBaudiotoolboxmidi)
+
 liblpcm_plugin_la_SOURCES = codec/lpcm.c
 codec_LTLIBRARIES += liblpcm_plugin.la
 
diff --git a/modules/codec/audiotoolbox_midi.c b/modules/codec/audiotoolbox_midi.c
new file mode 100644
index 0000000000..229df38328
--- /dev/null
+++ b/modules/codec/audiotoolbox_midi.c
@@ -0,0 +1,403 @@
+/*****************************************************************************
+ * audiotoolbox_midi.c: Software MIDI synthesizer using AudioToolbox
+ *****************************************************************************
+ * Copyright (C) 2017 VLC authors and VideoLAN
+ * $Id$
+ *
+ * Authors: Marvin Scholz <epirat07 at gmail dot com>
+ *
+ * Based on the fluidsynth module by RĂ©mi Denis-Courmont
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <vlc_common.h>
+#include <vlc_plugin.h>
+#include <vlc_codec.h>
+#include <vlc_dialog.h>
+
+#include <CoreFoundation/CoreFoundation.h>
+#include <AudioUnit/AudioUnit.h>
+#include <AudioToolbox/AudioToolbox.h>
+
+#include <TargetConditionals.h>
+
+#ifndef on_err_goto
+#define on_err_goto(errorCode, exceptionLabel)  \
+do { if ((errorCode) != noErr) goto exceptionLabel; \
+} while ( 0 )
+#endif
+
+#define SOUNDFONT_TEXT N_("Sound font file")
+#define SOUNDFONT_LONGTEXT N_( \
+    "The Sound Font file (SF2/DLS) to use for synthesis.")
+
+static int  Open  (vlc_object_t *);
+static void Close (vlc_object_t *);
+
+#define CFG_PREFIX "aumidi-"
+
+vlc_module_begin()
+    set_description(N_("AudioToolbox MIDI synthesizer"))
+    set_capability("audio decoder", 100)
+    set_shortname(N_("AUMIDI"))
+    set_category(CAT_INPUT)
+    set_subcategory(SUBCAT_INPUT_ACODEC)
+    set_callbacks(Open, Close)
+    add_loadfile(CFG_PREFIX "soundfont", "",
+                 SOUNDFONT_TEXT, SOUNDFONT_LONGTEXT, false)
+vlc_module_end()
+
+
+struct decoder_sys_t
+{
+    AUGraph     graph;
+    AudioUnit   synthUnit;
+    AudioUnit   outputUnit;
+    date_t       end_date;
+};
+
+static int  DecodeBlock (decoder_t *p_dec, block_t *p_block);
+static void Flush (decoder_t *);
+
+/* MIDI constants */
+enum
+{
+    kMidiMessage_NoteOff            = 0x80,
+    kMidiMessage_NoteOn             = 0x90,
+    kMidiMessage_PolyPressure       = 0xA0,
+    kMidiMessage_ControlChange      = 0xB0,
+    kMidiMessage_ProgramChange      = 0xC0,
+    kMidiMessage_ChannelPressure    = 0xD0,
+    kMidiMessage_PitchWheel         = 0xE0,
+    kMidiMessage_SysEx              = 0xF0,
+ 
+    kMidiMessage_BankMSBControl     = 0,
+    kMidiMessage_BankLSBControl     = 32,
+
+    /* Values for kMidiMessage_ControlChange */
+    kMidiController_AllSoundOff         = 0x78,
+    kMidiController_ResetAllControllers = 0x79,
+    kMidiController_AllNotesOff         = 0x7B
+};
+
+/* Helper functions */
+static OSStatus AddAppleAUNode(AUGraph graph, OSType type, OSType subtype, AUNode *node)
+{
+    AudioComponentDescription cDesc = {};
+    cDesc.componentType = type;
+    cDesc.componentSubType = subtype;
+    cDesc.componentManufacturer = kAudioUnitManufacturer_Apple;
+
+    return AUGraphAddNode(graph, &cDesc, node);
+}
+
+static OSStatus CreateAUGraph(AUGraph *outGraph, AudioUnit *outSynth, AudioUnit *outOut)
+{
+    OSStatus res;
+    
+    // AudioUnit nodes
+    AUNode synthNode, limiterNode, outNode;
+
+    // Create the Graph to which we will add our nodes
+    on_err_goto(res = NewAUGraph(outGraph), bailout);
+
+    // Create/add the MIDI synthesizer node (DLS Synth)
+#if TARGET_OS_IPHONE
+    // On iOS/tvOS use MIDISynth, DLSSynth does not exist there
+    on_err_goto(res = AddAppleAUNode(*outGraph, 
+                                     kAudioUnitType_MusicDevice, 
+                                     kAudioUnitSubType_MIDISynth,
+                                     &synthNode), bailout);
+#else
+    // Prefer DLSSynth on macOS, as it has a better default behavior
+    on_err_goto(res = AddAppleAUNode(*outGraph, 
+                                     kAudioUnitType_MusicDevice, 
+                                     kAudioUnitSubType_DLSSynth, 
+                                     &synthNode), bailout);
+#endif
+
+    // Create/add the peak limiter node
+    on_err_goto(res = AddAppleAUNode(*outGraph, 
+                                     kAudioUnitType_Effect, 
+                                     kAudioUnitSubType_PeakLimiter, 
+                                     &limiterNode), bailout);
+
+    // Create/add the output node (GenericOutput)
+    on_err_goto(res = AddAppleAUNode(*outGraph, 
+                                     kAudioUnitType_Output, 
+                                     kAudioUnitSubType_GenericOutput, 
+                                     &outNode), bailout);
+    
+    // Open the Graph, this opens the units that belong to the graph
+    // so that we can connect them
+    on_err_goto(res = AUGraphOpen(*outGraph), bailout);
+
+    // Connect the synthesizer node to the limiter
+    on_err_goto(res = AUGraphConnectNodeInput(*outGraph, synthNode, 0, limiterNode, 0), bailout);
+    // Connect the limiter node to the output
+    on_err_goto(res = AUGraphConnectNodeInput(*outGraph, limiterNode, 0, outNode, 0), bailout);
+    
+    // Get reference to the synthesizer node
+    on_err_goto(res = AUGraphNodeInfo(*outGraph, synthNode, 0, outSynth), bailout);
+    // Get reference to the output node
+    on_err_goto(res = AUGraphNodeInfo(*outGraph, outNode, 0, outOut), bailout);
+
+bailout:
+    return res;
+}
+
+static int SetSoundfont(decoder_t *p_dec, AudioUnit synthUnit, const char *sfPath) {
+    if (!sfPath) {
+        msg_Dbg(p_dec, "using default soundfont");
+        return VLC_SUCCESS;
+    }
+    
+    msg_Dbg(p_dec, "using custom soundfont: '%s'", sfPath);
+    CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault,
+                                                           (const UInt8 *)sfPath,
+                                                           strlen(sfPath), false);
+    if (unlikely(url == NULL))
+        return VLC_ENOMEM;
+    
+    OSStatus status = AudioUnitSetProperty(synthUnit,
+                                           kMusicDeviceProperty_SoundBankURL,
+                                           kAudioUnitScope_Global, 0,
+                                           &url, sizeof(url));
+    CFRelease(url);
+    
+    if (status != noErr) {
+        msg_Err(p_dec, "failed setting custom SoundFont for MIDI synthesis (%i)", status);
+        return VLC_EGENERIC;
+    }
+    return VLC_SUCCESS;
+}
+
+static int Open(vlc_object_t *p_this)
+{
+    decoder_t *p_dec = (decoder_t *)p_this;
+    OSStatus status = noErr;
+
+    if (p_dec->fmt_in.i_codec != VLC_CODEC_MIDI)
+        return VLC_EGENERIC;
+
+    decoder_sys_t *p_sys = malloc(sizeof (*p_sys));
+    if (unlikely(p_sys == NULL))
+        return VLC_ENOMEM;
+
+    status = CreateAUGraph(&p_sys->graph, &p_sys->synthUnit, &p_sys->outputUnit);
+    if (unlikely(status != noErr)) {
+        msg_Err(p_dec, "failed to create audiograph (%i)", status);
+        return VLC_EGENERIC;
+    }
+
+    // Set custom soundfont
+    char *sfPath = var_InheritString(p_dec, CFG_PREFIX "soundfont");
+    int ret = SetSoundfont(p_dec, p_sys->synthUnit, sfPath);
+    free(sfPath);
+    if (unlikely(ret != VLC_SUCCESS))
+        return ret;
+
+    // Prepare AudioUnit output audio format info
+    AudioStreamBasicDescription ASBD = {};
+    unsigned bytesPerSample = sizeof(Float32);
+    ASBD.mFormatID = kAudioFormatLinearPCM;
+    ASBD.mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked;
+    ASBD.mSampleRate = 44100;
+    ASBD.mFramesPerPacket = 1;
+    ASBD.mChannelsPerFrame = 2;
+    ASBD.mBytesPerFrame = bytesPerSample * ASBD.mChannelsPerFrame;
+    ASBD.mBytesPerPacket = ASBD.mBytesPerFrame * ASBD.mFramesPerPacket;
+    ASBD.mBitsPerChannel = 8 * bytesPerSample;
+
+    // Set AudioUnit format
+    status = AudioUnitSetProperty(p_sys->outputUnit, 
+                                  kAudioUnitProperty_StreamFormat, 
+                                  kAudioUnitScope_Output, 0, &ASBD,
+                                  sizeof(AudioStreamBasicDescription));
+    if (unlikely(status != noErr)) {
+        msg_Err(p_dec, "failed setting output format for output unit (%i)", status);
+        return VLC_EGENERIC;
+    }
+
+    // Prepare the AU
+    status = AUGraphInitialize (p_sys->graph);
+    if (unlikely(status != noErr)) {
+        if (status == kAudioUnitErr_InvalidFile)
+            msg_Err(p_dec, "failed initializing audiograph: invalid soundfont file");
+        else
+            msg_Err(p_dec, "failed initializing audiograph (%i)", status);
+        return VLC_EGENERIC;
+    }
+
+    // Prepare MIDI soundbank
+    MusicDeviceMIDIEvent(p_sys->synthUnit, 
+                         kMidiMessage_ControlChange, 
+                         kMidiMessage_BankMSBControl, 0, 0);
+
+    // Start the AU
+    status = AUGraphStart(p_sys->graph);
+    if (unlikely(status != noErr)) {
+        msg_Err(p_dec, "failed starting audiograph (%i)", status);
+        return VLC_EGENERIC;
+    }
+
+    // Set VLC output audio format info
+    p_dec->fmt_out.i_codec = VLC_CODEC_FL32;
+    p_dec->fmt_out.audio.i_bitspersample = 32;
+    p_dec->fmt_out.audio.i_rate = 44100;
+    p_dec->fmt_out.audio.i_channels = 2;
+    p_dec->fmt_out.audio.i_physical_channels = AOUT_CHAN_LEFT | AOUT_CHAN_RIGHT;
+
+    // Initialize date (for PTS)
+    date_Init(&p_sys->end_date, p_dec->fmt_out.audio.i_rate, 1);
+    date_Set(&p_sys->end_date, 0);
+
+    p_dec->p_sys = p_sys;
+    p_dec->pf_decode = DecodeBlock;
+    p_dec->pf_flush  = Flush;
+    return VLC_SUCCESS;
+}
+
+
+static void Close (vlc_object_t *p_this)
+{
+    decoder_sys_t *p_sys = ((decoder_t *)p_this)->p_sys;
+
+    if (p_sys->graph) {
+        AUGraphStop(p_sys->graph);
+        DisposeAUGraph(p_sys->graph);
+    }
+    free(p_sys);
+}
+
+static void Flush (decoder_t *p_dec)
+{
+    decoder_sys_t *p_sys = p_dec->p_sys;
+
+    date_Set(&p_sys->end_date, VLC_TS_INVALID);
+
+    // Turn all sound on all channels off
+    // else 'old' notes could still be playing
+    for (unsigned channel = 0; channel < 16; channel++) {
+        MusicDeviceMIDIEvent(p_sys->synthUnit, kMidiMessage_ControlChange | channel, kMidiController_AllSoundOff, 0, 0);
+    }
+}
+
+static int DecodeBlock (decoder_t *p_dec, block_t *p_block)
+{
+    decoder_sys_t *p_sys = p_dec->p_sys;
+    block_t *p_out = NULL;
+    OSStatus status = noErr;
+
+    if (p_block == NULL) /* No Drain */
+        return VLCDEC_SUCCESS;
+
+    if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
+        Flush(p_dec);
+        if (p_block->i_flags & BLOCK_FLAG_CORRUPTED) {
+            block_Release(p_block);
+            return VLCDEC_SUCCESS;
+        }
+    }
+
+    if (p_block->i_pts > VLC_TS_INVALID && !date_Get(&p_sys->end_date)) {
+        date_Set(&p_sys->end_date, p_block->i_pts);
+    } else if (p_block->i_pts < date_Get(&p_sys->end_date)) {
+        msg_Warn(p_dec, "MIDI message in the past?");
+        goto drop;
+    }
+
+    if (p_block->i_buffer < 1)
+        goto drop;
+
+    uint8_t event = p_block->p_buffer[0];
+    uint8_t data1 = (p_block->i_buffer > 1) ? (p_block->p_buffer[1]) : 0;
+    uint8_t data2 = (p_block->i_buffer > 2) ? (p_block->p_buffer[2]) : 0;
+
+    switch (event & 0xF0)
+    {
+        case kMidiMessage_NoteOff:
+        case kMidiMessage_NoteOn:
+        case kMidiMessage_PolyPressure:
+        case kMidiMessage_ControlChange:
+        case kMidiMessage_ProgramChange:
+        case kMidiMessage_ChannelPressure:
+        case kMidiMessage_PitchWheel:
+            MusicDeviceMIDIEvent(p_sys->synthUnit, event, data1, data2, 0);
+        break;
+
+        case kMidiMessage_SysEx:
+            // SysEx not handled
+        break;
+
+        default:
+            msg_Warn(p_dec, "unhandled MIDI event: %x", event & 0xF0);
+        break;
+    }
+
+    // Calculate frame count
+    // Simplification of 44100 / 1000000
+    // TODO: Other samplerates
+    unsigned frames =
+       (p_block->i_pts - date_Get(&p_sys->end_date)) * 441 / 10000;
+
+    if (frames == 0)
+        goto drop;
+
+    if (decoder_UpdateAudioFormat(p_dec))
+        goto drop;
+
+    p_out = decoder_NewAudioBuffer(p_dec, frames);
+    if (p_out == NULL)
+        goto drop;
+
+    p_out->i_pts = date_Get(&p_sys->end_date );
+    p_out->i_length = date_Increment(&p_sys->end_date, frames)
+                      - p_out->i_pts;
+
+    // Prepare Timestamp for the AudioUnit render call
+    AudioTimeStamp timestamp = {};
+    timestamp.mFlags = kAudioTimeStampWordClockTimeValid;
+    timestamp.mWordClockTime = p_out->i_pts;
+
+    // Prepare Buffer for the AudioUnit render call
+    AudioBufferList bufferList;
+    bufferList.mNumberBuffers = 1;
+    bufferList.mBuffers[0].mNumberChannels = 2;
+    bufferList.mBuffers[0].mDataByteSize = frames * sizeof(Float32) * 2;
+    bufferList.mBuffers[0].mData = p_out->p_buffer;
+
+    status = AudioUnitRender(p_sys->outputUnit,
+                             NULL,
+                             &timestamp, 0,
+                             frames, &bufferList);
+
+    if (status != noErr) {
+        msg_Warn(p_dec, "rendering audio unit failed: %i", status);
+        block_Release(p_out);
+        p_out = NULL;
+    }
+
+drop:
+    block_Release(p_block);
+    if (p_out != NULL)
+        decoder_QueueAudio(p_dec, p_out);
+    return VLCDEC_SUCCESS;
+}
diff --git a/po/POTFILES.in b/po/POTFILES.in
index 040e65cb10..c1a1facd88 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -292,6 +292,7 @@ modules/codec/adpcm.c
 modules/codec/aes3.c
 modules/codec/aom.c
 modules/codec/araw.c
+modules/codec/audiotoolbox_midi.c
 modules/codec/arib/aribsub.c
 modules/codec/avcodec/avcodec.c
 modules/codec/avcodec/avcodec.h
-- 
2.13.6 (Apple Git-96)



More information about the vlc-devel mailing list