[vlc-devel] [PATCH v3] codec/audiotoolbox_midi: Add AudioToolbox MIDI decoder

Marvin Scholz epirat07 at gmail.com
Wed Nov 29 16:40:38 CET 2017



On 29 Nov 2017, at 16:34, Thomas Guillem wrote:

> On Wed, Nov 29, 2017, at 16:22, Marvin Scholz wrote:
>> ---
>>  NEWS                              |   2 +
>>  configure.ac                      |   6 +
>>  modules/MODULES_LIST              |   3 +-
>>  modules/codec/Makefile.am         |   6 +
>>  modules/codec/audiotoolbox_midi.c | 403
>>  ++++++++++++++++++++++++++++++++++++++
>>  po/POTFILES.in                    |   1 +
>>  6 files changed, 420 insertions(+), 1 deletion(-)
>>  create mode 100644 modules/codec/audiotoolbox_midi.c
>>
>> diff --git a/NEWS b/NEWS
>> index ddd1626f37..edc5d61d37 100644
>> --- a/NEWS
>> +++ b/NEWS
>> @@ -99,6 +99,8 @@ Decoder:
>>   * Extend MicroDVD support with color, fontname, size, position
>>   extensions
>>   * BluRay text subtitles are now decoded
>>   * Improved Closed Captions detection and optional CEA-708 decoder
>> + * New MIDI decoder for macOS and iOS using the AudioToolbox 
>> framework,
>> works
>> +   without a soundfont or with SoundFont2 and DLS soundfonts
>>
>>  Demuxers:
>>   * Support HD-DVD .evo (H.264, VC-1, MPEG-2, PCM, AC-3, E-AC3, MLP, 
>> DTS)
>> diff --git a/configure.ac b/configure.ac
>> index 62ce59aff5..fc4e05f9ed 100644
>> --- a/configure.ac
>> +++ b/configure.ac
>> @@ -3894,6 +3894,12 @@ AC_CHECK_HEADERS(VideoToolbox/VideoToolbox.h, 
>> [
>>    ])
>>
>>  dnl
>> +dnl  AudioToolbox MIDI plugin
>> +AC_CHECK_HEADERS([AudioToolbox/AudioToolbox.h], [
>> +  VLC_ADD_PLUGIN([audiotoolboxmidi])
>> +])
>> +
>> +dnl
>>  dnl  ncurses module
>>  dnl
>>  AC_ARG_ENABLE(ncurses,
>> diff --git a/modules/MODULES_LIST b/modules/MODULES_LIST
>> index 014c5ddafb..146e537a56 100644
>> --- a/modules/MODULES_LIST
>> +++ b/modules/MODULES_LIST
>> @@ -1,4 +1,4 @@
>> -List of vlc plugins (490):
>> +List of vlc plugins (491):
>>  $Id$
>>
>>   * a52: A/52 audio decoder plugin, using liba52
>> @@ -53,6 +53,7 @@ $Id$
>>   * audiobargraph_a: audiobargraph audio plugin
>>   * audiobargraph_v: audiobargraph video plugin
>>   * audioscrobbler: AudioScrobbler/Last.fm submission plugin
>> + * audiotoolbox_midi: AudioToolbox MIDI decoder plugin for macOS
>>   * audiounit_ios: AudioUnit output plugin for iOS
>>   * auhal: Audio output for Mac OS X based on the AUHAL API
>>   * avahi: Zeroconf services discovery
>> diff --git a/modules/codec/Makefile.am b/modules/codec/Makefile.am
>> index 075ee2a0e9..6a8a72019c 100644
>> --- a/modules/codec/Makefile.am
>> +++ b/modules/codec/Makefile.am
>> @@ -60,6 +60,12 @@ if HAVE_DARWIN
>>  libfluidsynth_plugin_la_LDFLAGS +=
>>  -Wl,-framework,CoreFoundation,-framework,CoreServices
>>  endif
>>
>> +libaudiotoolboxmidi_plugin_la_SOURCES = codec/audiotoolbox_midi.c
>> +libaudiotoolboxmidi_plugin_la_LDFLAGS = $(AM_LDFLAGS) -rpath
>> '$(codecdir)'
>> +libaudiotoolboxmidi_plugin_la_LDFLAGS +=
>> -Wl,-framework,CoreFoundation,-framework,AudioUnit,-framework,AudioToolbox
>> +EXTRA_LTLIBRARIES += libaudiotoolboxmidi_plugin.la
>> +codec_LTLIBRARIES += $(LTLIBaudiotoolboxmidi)
>> +
>>  liblpcm_plugin_la_SOURCES = codec/lpcm.c
>>  codec_LTLIBRARIES += liblpcm_plugin.la
>>
>> diff --git a/modules/codec/audiotoolbox_midi.c
>> b/modules/codec/audiotoolbox_midi.c
>> new file mode 100644
>> index 0000000000..229df38328
>> --- /dev/null
>> +++ b/modules/codec/audiotoolbox_midi.c
>> @@ -0,0 +1,403 @@
>> +/*****************************************************************************
>> + * audiotoolbox_midi.c: Software MIDI synthesizer using AudioToolbox
>> +
>> *****************************************************************************
>> + * Copyright (C) 2017 VLC authors and VideoLAN
>> + * $Id$
>> + *
>> + * Authors: Marvin Scholz <epirat07 at gmail dot com>
>> + *
>> + * Based on the fluidsynth module by RĂ©mi Denis-Courmont
>> + *
>> + * This program is free software; you can redistribute it and/or 
>> modify
>> it
>> + * under the terms of the GNU Lesser General Public License as 
>> published
>> by
>> + * the Free Software Foundation; either version 2.1 of the License, 
>> or
>> + * (at your option) any later version.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>> + * GNU Lesser General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU Lesser General Public
>> License
>> + * along with this program; if not, write to the Free Software
>> Foundation,
>> + * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
>> +
>> *****************************************************************************/
>> +
>> +#ifdef HAVE_CONFIG_H
>> +# include "config.h"
>> +#endif
>> +
>> +#include <vlc_common.h>
>> +#include <vlc_plugin.h>
>> +#include <vlc_codec.h>
>> +#include <vlc_dialog.h>
>> +
>> +#include <CoreFoundation/CoreFoundation.h>
>> +#include <AudioUnit/AudioUnit.h>
>> +#include <AudioToolbox/AudioToolbox.h>
>> +
>> +#include <TargetConditionals.h>
>> +
>> +#ifndef on_err_goto
>> +#define on_err_goto(errorCode, exceptionLabel)  \
>> +do { if ((errorCode) != noErr) goto exceptionLabel; \
>> +} while ( 0 )
>> +#endif
>> +
>> +#define SOUNDFONT_TEXT N_("Sound font file")
>> +#define SOUNDFONT_LONGTEXT N_( \
>> +    "The Sound Font file (SF2/DLS) to use for synthesis.")
>> +
>> +static int  Open  (vlc_object_t *);
>> +static void Close (vlc_object_t *);
>> +
>> +#define CFG_PREFIX "aumidi-"
>> +
>> +vlc_module_begin()
>> +    set_description(N_("AudioToolbox MIDI synthesizer"))
>> +    set_capability("audio decoder", 100)
>> +    set_shortname(N_("AUMIDI"))
>> +    set_category(CAT_INPUT)
>> +    set_subcategory(SUBCAT_INPUT_ACODEC)
>> +    set_callbacks(Open, Close)
>> +    add_loadfile(CFG_PREFIX "soundfont", "",
>> +                 SOUNDFONT_TEXT, SOUNDFONT_LONGTEXT, false)
>> +vlc_module_end()
>> +
>> +
>> +struct decoder_sys_t
>> +{
>> +    AUGraph     graph;
>> +    AudioUnit   synthUnit;
>> +    AudioUnit   outputUnit;
>> +    date_t       end_date;
>> +};
>> +
>> +static int  DecodeBlock (decoder_t *p_dec, block_t *p_block);
>> +static void Flush (decoder_t *);
>> +
>> +/* MIDI constants */
>> +enum
>> +{
>> +    kMidiMessage_NoteOff            = 0x80,
>> +    kMidiMessage_NoteOn             = 0x90,
>> +    kMidiMessage_PolyPressure       = 0xA0,
>> +    kMidiMessage_ControlChange      = 0xB0,
>> +    kMidiMessage_ProgramChange      = 0xC0,
>> +    kMidiMessage_ChannelPressure    = 0xD0,
>> +    kMidiMessage_PitchWheel         = 0xE0,
>> +    kMidiMessage_SysEx              = 0xF0,
>> +
>> +    kMidiMessage_BankMSBControl     = 0,
>> +    kMidiMessage_BankLSBControl     = 32,
>> +
>> +    /* Values for kMidiMessage_ControlChange */
>> +    kMidiController_AllSoundOff         = 0x78,
>> +    kMidiController_ResetAllControllers = 0x79,
>> +    kMidiController_AllNotesOff         = 0x7B
>> +};
>> +
>> +/* Helper functions */
>> +static OSStatus AddAppleAUNode(AUGraph graph, OSType type, OSType
>> subtype, AUNode *node)
>> +{
>> +    AudioComponentDescription cDesc = {};
>> +    cDesc.componentType = type;
>> +    cDesc.componentSubType = subtype;
>> +    cDesc.componentManufacturer = kAudioUnitManufacturer_Apple;
>> +
>> +    return AUGraphAddNode(graph, &cDesc, node);
>> +}
>> +
>> +static OSStatus CreateAUGraph(AUGraph *outGraph, AudioUnit 
>> *outSynth,
>> AudioUnit *outOut)
>> +{
>> +    OSStatus res;
>> +
>> +    // AudioUnit nodes
>> +    AUNode synthNode, limiterNode, outNode;
>> +
>> +    // Create the Graph to which we will add our nodes
>> +    on_err_goto(res = NewAUGraph(outGraph), bailout);
>> +
>> +    // Create/add the MIDI synthesizer node (DLS Synth)
>> +#if TARGET_OS_IPHONE
>> +    // On iOS/tvOS use MIDISynth, DLSSynth does not exist there
>> +    on_err_goto(res = AddAppleAUNode(*outGraph,
>> +                                     kAudioUnitType_MusicDevice,
>> +                                     kAudioUnitSubType_MIDISynth,
>> +                                     &synthNode), bailout);
>> +#else
>> +    // Prefer DLSSynth on macOS, as it has a better default behavior
>> +    on_err_goto(res = AddAppleAUNode(*outGraph,
>> +                                     kAudioUnitType_MusicDevice,
>> +                                     kAudioUnitSubType_DLSSynth,
>> +                                     &synthNode), bailout);
>> +#endif
>> +
>> +    // Create/add the peak limiter node
>> +    on_err_goto(res = AddAppleAUNode(*outGraph,
>> +                                     kAudioUnitType_Effect,
>> +                                     kAudioUnitSubType_PeakLimiter,
>> +                                     &limiterNode), bailout);
>> +
>> +    // Create/add the output node (GenericOutput)
>> +    on_err_goto(res = AddAppleAUNode(*outGraph,
>> +                                     kAudioUnitType_Output,
>> +                                     
>> kAudioUnitSubType_GenericOutput,
>> +                                     &outNode), bailout);
>> +
>> +    // Open the Graph, this opens the units that belong to the graph
>> +    // so that we can connect them
>> +    on_err_goto(res = AUGraphOpen(*outGraph), bailout);
>> +
>> +    // Connect the synthesizer node to the limiter
>> +    on_err_goto(res = AUGraphConnectNodeInput(*outGraph, synthNode, 
>> 0,
>> limiterNode, 0), bailout);
>> +    // Connect the limiter node to the output
>> +    on_err_goto(res = AUGraphConnectNodeInput(*outGraph, 
>> limiterNode, 0,
>> outNode, 0), bailout);
>> +
>> +    // Get reference to the synthesizer node
>> +    on_err_goto(res = AUGraphNodeInfo(*outGraph, synthNode, 0,
>> outSynth), bailout);
>> +    // Get reference to the output node
>> +    on_err_goto(res = AUGraphNodeInfo(*outGraph, outNode, 0, 
>> outOut),
>> bailout);
>> +
>> +bailout:
>> +    return res;
>> +}
>> +
>> +static int SetSoundfont(decoder_t *p_dec, AudioUnit synthUnit, const
>> char *sfPath) {
>> +    if (!sfPath) {
>> +        msg_Dbg(p_dec, "using default soundfont");
>> +        return VLC_SUCCESS;
>> +    }
>> +
>> +    msg_Dbg(p_dec, "using custom soundfont: '%s'", sfPath);
>> +    CFURLRef url =
>> CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault,
>> +                                                           (const 
>> UInt8
>> *)sfPath,
>> +
>> strlen(sfPath), false);
>> +    if (unlikely(url == NULL))
>> +        return VLC_ENOMEM;
>> +
>> +    OSStatus status = AudioUnitSetProperty(synthUnit,
>> +
>> kMusicDeviceProperty_SoundBankURL,
>> +                                           kAudioUnitScope_Global, 
>> 0,
>> +                                           &url, sizeof(url));
>> +    CFRelease(url);
>> +
>> +    if (status != noErr) {
>> +        msg_Err(p_dec, "failed setting custom SoundFont for MIDI
>> synthesis (%i)", status);
>> +        return VLC_EGENERIC;
>> +    }
>> +    return VLC_SUCCESS;
>> +}
>> +
>> +static int Open(vlc_object_t *p_this)
>> +{
>> +    decoder_t *p_dec = (decoder_t *)p_this;
>> +    OSStatus status = noErr;
>> +
>> +    if (p_dec->fmt_in.i_codec != VLC_CODEC_MIDI)
>> +        return VLC_EGENERIC;
>> +
>> +    decoder_sys_t *p_sys = malloc(sizeof (*p_sys));
>> +    if (unlikely(p_sys == NULL))
>> +        return VLC_ENOMEM;
>> +
>> +    status = CreateAUGraph(&p_sys->graph, &p_sys->synthUnit,
>> &p_sys->outputUnit);
>> +    if (unlikely(status != noErr)) {
>> +        msg_Err(p_dec, "failed to create audiograph (%i)", status);
>> +        return VLC_EGENERIC;
>> +    }
>> +
>> +    // Set custom soundfont
>> +    char *sfPath = var_InheritString(p_dec, CFG_PREFIX "soundfont");
>> +    int ret = SetSoundfont(p_dec, p_sys->synthUnit, sfPath);
>> +    free(sfPath);
>> +    if (unlikely(ret != VLC_SUCCESS))
>> +        return ret;
>> +
>> +    // Prepare AudioUnit output audio format info
>> +    AudioStreamBasicDescription ASBD = {};
>> +    unsigned bytesPerSample = sizeof(Float32);
>> +    ASBD.mFormatID = kAudioFormatLinearPCM;
>> +    ASBD.mFormatFlags = kAudioFormatFlagIsFloat |
>> kAudioFormatFlagIsPacked;
>> +    ASBD.mSampleRate = 44100;
>> +    ASBD.mFramesPerPacket = 1;
>> +    ASBD.mChannelsPerFrame = 2;
>> +    ASBD.mBytesPerFrame = bytesPerSample * ASBD.mChannelsPerFrame;
>> +    ASBD.mBytesPerPacket = ASBD.mBytesPerFrame * 
>> ASBD.mFramesPerPacket;
>> +    ASBD.mBitsPerChannel = 8 * bytesPerSample;
>> +
>> +    // Set AudioUnit format
>> +    status = AudioUnitSetProperty(p_sys->outputUnit,
>> +                                  kAudioUnitProperty_StreamFormat,
>> +                                  kAudioUnitScope_Output, 0, &ASBD,
>> +                                  
>> sizeof(AudioStreamBasicDescription));
>> +    if (unlikely(status != noErr)) {
>> +        msg_Err(p_dec, "failed setting output format for output unit
>> (%i)", status);
>> +        return VLC_EGENERIC;
>> +    }
>> +
>> +    // Prepare the AU
>> +    status = AUGraphInitialize (p_sys->graph);
>> +    if (unlikely(status != noErr)) {
>> +        if (status == kAudioUnitErr_InvalidFile)
>> +            msg_Err(p_dec, "failed initializing audiograph: invalid
>> soundfont file");
>> +        else
>> +            msg_Err(p_dec, "failed initializing audiograph (%i)",
>> status);
>> +        return VLC_EGENERIC;
>> +    }
>> +
>> +    // Prepare MIDI soundbank
>> +    MusicDeviceMIDIEvent(p_sys->synthUnit,
>> +                         kMidiMessage_ControlChange,
>> +                         kMidiMessage_BankMSBControl, 0, 0);
>> +
>> +    // Start the AU
>> +    status = AUGraphStart(p_sys->graph);
>> +    if (unlikely(status != noErr)) {
>> +        msg_Err(p_dec, "failed starting audiograph (%i)", status);
>> +        return VLC_EGENERIC;
>> +    }
>> +
>> +    // Set VLC output audio format info
>> +    p_dec->fmt_out.i_codec = VLC_CODEC_FL32;
>> +    p_dec->fmt_out.audio.i_bitspersample = 32;
>> +    p_dec->fmt_out.audio.i_rate = 44100;
>> +    p_dec->fmt_out.audio.i_channels = 2;
>> +    p_dec->fmt_out.audio.i_physical_channels = AOUT_CHAN_LEFT |
>> AOUT_CHAN_RIGHT;
>> +
>> +    // Initialize date (for PTS)
>> +    date_Init(&p_sys->end_date, p_dec->fmt_out.audio.i_rate, 1);
>> +    date_Set(&p_sys->end_date, 0);
>> +
>> +    p_dec->p_sys = p_sys;
>> +    p_dec->pf_decode = DecodeBlock;
>> +    p_dec->pf_flush  = Flush;
>> +    return VLC_SUCCESS;
>> +}
>> +
>> +
>> +static void Close (vlc_object_t *p_this)
>> +{
>> +    decoder_sys_t *p_sys = ((decoder_t *)p_this)->p_sys;
>> +
>> +    if (p_sys->graph) {
>> +        AUGraphStop(p_sys->graph);
>> +        DisposeAUGraph(p_sys->graph);
>> +    }
>> +    free(p_sys);
>> +}
>> +
>> +static void Flush (decoder_t *p_dec)
>> +{
>> +    decoder_sys_t *p_sys = p_dec->p_sys;
>> +
>> +    date_Set(&p_sys->end_date, VLC_TS_INVALID);
>> +
>> +    // Turn all sound on all channels off
>> +    // else 'old' notes could still be playing
>> +    for (unsigned channel = 0; channel < 16; channel++) {
>> +        MusicDeviceMIDIEvent(p_sys->synthUnit,
>> kMidiMessage_ControlChange | channel, kMidiController_AllSoundOff, 0, 
>> 0);
>> +    }
>> +}
>> +
>> +static int DecodeBlock (decoder_t *p_dec, block_t *p_block)
>> +{
>> +    decoder_sys_t *p_sys = p_dec->p_sys;
>> +    block_t *p_out = NULL;
>> +    OSStatus status = noErr;
>> +
>> +    if (p_block == NULL) /* No Drain */
>> +        return VLCDEC_SUCCESS;
>> +
>> +    if (p_block->i_flags &
>> (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
>> +        Flush(p_dec);
>> +        if (p_block->i_flags & BLOCK_FLAG_CORRUPTED) {
>> +            block_Release(p_block);
>> +            return VLCDEC_SUCCESS;
>> +        }
>> +    }
>> +
>> +    if (p_block->i_pts > VLC_TS_INVALID && 
>> !date_Get(&p_sys->end_date))
>> {
>> +        date_Set(&p_sys->end_date, p_block->i_pts);
>> +    } else if (p_block->i_pts < date_Get(&p_sys->end_date)) {
>> +        msg_Warn(p_dec, "MIDI message in the past?");
>> +        goto drop;
>> +    }
>> +
>> +    if (p_block->i_buffer < 1)
>> +        goto drop;
>> +
>> +    uint8_t event = p_block->p_buffer[0];
>> +    uint8_t data1 = (p_block->i_buffer > 1) ? (p_block->p_buffer[1]) 
>> :
>> 0;
>> +    uint8_t data2 = (p_block->i_buffer > 2) ? (p_block->p_buffer[2]) 
>> :
>> 0;
>> +
>> +    switch (event & 0xF0)
>> +    {
>> +        case kMidiMessage_NoteOff:
>> +        case kMidiMessage_NoteOn:
>> +        case kMidiMessage_PolyPressure:
>> +        case kMidiMessage_ControlChange:
>> +        case kMidiMessage_ProgramChange:
>> +        case kMidiMessage_ChannelPressure:
>> +        case kMidiMessage_PitchWheel:
>> +            MusicDeviceMIDIEvent(p_sys->synthUnit, event, data1, 
>> data2,
>> 0);
>> +        break;
>> +
>> +        case kMidiMessage_SysEx:
>> +            // SysEx not handled
>> +        break;
>> +
>> +        default:
>> +            msg_Warn(p_dec, "unhandled MIDI event: %x", event & 
>> 0xF0);
>> +        break;
>> +    }
>> +
>> +    // Calculate frame count
>> +    // Simplification of 44100 / 1000000
>> +    // TODO: Other samplerates
>> +    unsigned frames =
>> +       (p_block->i_pts - date_Get(&p_sys->end_date)) * 441 / 10000;
>> +
>> +    if (frames == 0)
>> +        goto drop;
>> +
>> +    if (decoder_UpdateAudioFormat(p_dec))
>> +        goto drop;
>
> What about my first comment?
>
> Ok I wrote a typo: here is my fixed comment again:
>
> If dec->fmt_out *can not* be changed while decoding, you should put 
> this
> code
> in the Open() function just after setting fmt_out.
>

Oh sorry, completely forgot about this. Will send a new patch.
Thanks for reminding me!

>> +
>> +    p_out = decoder_NewAudioBuffer(p_dec, frames);
>> +    if (p_out == NULL)
>> +        goto drop;
>> +
>> +    p_out->i_pts = date_Get(&p_sys->end_date );
>> +    p_out->i_length = date_Increment(&p_sys->end_date, frames)
>> +                      - p_out->i_pts;
>> +
>> +    // Prepare Timestamp for the AudioUnit render call
>> +    AudioTimeStamp timestamp = {};
>> +    timestamp.mFlags = kAudioTimeStampWordClockTimeValid;
>> +    timestamp.mWordClockTime = p_out->i_pts;
>> +
>> +    // Prepare Buffer for the AudioUnit render call
>> +    AudioBufferList bufferList;
>> +    bufferList.mNumberBuffers = 1;
>> +    bufferList.mBuffers[0].mNumberChannels = 2;
>> +    bufferList.mBuffers[0].mDataByteSize = frames * sizeof(Float32) 
>> * 2;
>> +    bufferList.mBuffers[0].mData = p_out->p_buffer;
>> +
>> +    status = AudioUnitRender(p_sys->outputUnit,
>> +                             NULL,
>> +                             &timestamp, 0,
>> +                             frames, &bufferList);
>> +
>> +    if (status != noErr) {
>> +        msg_Warn(p_dec, "rendering audio unit failed: %i", status);
>> +        block_Release(p_out);
>> +        p_out = NULL;
>> +    }
>> +
>> +drop:
>> +    block_Release(p_block);
>> +    if (p_out != NULL)
>> +        decoder_QueueAudio(p_dec, p_out);
>> +    return VLCDEC_SUCCESS;
>> +}
>> diff --git a/po/POTFILES.in b/po/POTFILES.in
>> index 040e65cb10..c1a1facd88 100644
>> --- a/po/POTFILES.in
>> +++ b/po/POTFILES.in
>> @@ -292,6 +292,7 @@ modules/codec/adpcm.c
>>  modules/codec/aes3.c
>>  modules/codec/aom.c
>>  modules/codec/araw.c
>> +modules/codec/audiotoolbox_midi.c
>>  modules/codec/arib/aribsub.c
>>  modules/codec/avcodec/avcodec.c
>>  modules/codec/avcodec/avcodec.h
>> -- 
>> 2.13.6 (Apple Git-96)
>>
>> _______________________________________________
>> vlc-devel mailing list
>> To unsubscribe or modify your subscription options:
>> https://mailman.videolan.org/listinfo/vlc-devel
> _______________________________________________
> vlc-devel mailing list
> To unsubscribe or modify your subscription options:
> https://mailman.videolan.org/listinfo/vlc-devel


More information about the vlc-devel mailing list