[vlc-devel] [PATCH] avaudiocapture: Add audio capture module based on AVFoundation framework
david.fuhrmann at gmail.com
david.fuhrmann at gmail.com
Thu Nov 1 11:30:16 CET 2018
From: David Fuhrmann <dfuhrmann at videolan.org>
This module is a drop-in replacement for the deprecated QTSound module.
QTSound is based on QtKit framework, which is long deprecated and does
not even exist anymore in current SDKs.
It supports the same feature set as QTSound.
closes #20883
---
.../macosx/VLC.xcodeproj/project.pbxproj | 4 +
modules/MODULES_LIST | 1 +
modules/access/Makefile.am | 7 +
modules/access/avaudiocapture.m | 345 ++++++++++++++++++
po/POTFILES.in | 1 +
5 files changed, 358 insertions(+)
create mode 100644 modules/access/avaudiocapture.m
diff --git a/extras/package/macosx/VLC.xcodeproj/project.pbxproj b/extras/package/macosx/VLC.xcodeproj/project.pbxproj
index 7dcbd72816d..84f28146d75 100644
--- a/extras/package/macosx/VLC.xcodeproj/project.pbxproj
+++ b/extras/package/macosx/VLC.xcodeproj/project.pbxproj
@@ -87,6 +87,7 @@
1CCC89042078A3D500E5626F /* StreamOutput.xib in Sources */ = {isa = PBXBuildFile; fileRef = 6B8224131E4D2A9000833BE1 /* StreamOutput.xib */; };
1CCC89052078A3D500E5626F /* TextfieldPanel.xib in Sources */ = {isa = PBXBuildFile; fileRef = 6B8224151E4D2A9000833BE1 /* TextfieldPanel.xib */; };
1CCC89062078A3D500E5626F /* TimeSelectionPanel.xib in Sources */ = {isa = PBXBuildFile; fileRef = 6B8224161E4D2A9000833BE1 /* TimeSelectionPanel.xib */; };
+ 1CCD8FA02117957700407114 /* avaudiocapture.m in Sources */ = {isa = PBXBuildFile; fileRef = 1CCD8F9F2117957600407114 /* avaudiocapture.m */; };
1CFE8D591EA0D42A00E94451 /* VLCErrorWindowController.m in Sources */ = {isa = PBXBuildFile; fileRef = 1CFE8D581EA0D42A00E94451 /* VLCErrorWindowController.m */; };
6B0292E61F43256300A50082 /* VLCBottomBarView.m in Sources */ = {isa = PBXBuildFile; fileRef = 6B0292E51F43256300A50082 /* VLCBottomBarView.m */; };
6B0AB0F01F1AC8B3003A1B4E /* VLCSlider.m in Sources */ = {isa = PBXBuildFile; fileRef = 6B0AB0ED1F1AC8B3003A1B4E /* VLCSlider.m */; };
@@ -167,6 +168,7 @@
1CAEBC001E1EC0A400A99E49 /* VLCFSPanelDraggableView.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = VLCFSPanelDraggableView.m; sourceTree = "<group>"; };
1CC25CA71B2C585D0003F994 /* darwinvlc.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = darwinvlc.m; path = ../../../bin/darwinvlc.m; sourceTree = "<group>"; };
1CCB5F2F1A62A6A5004C3E90 /* Pseudo-VLC.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Pseudo-VLC.app"; sourceTree = BUILT_PRODUCTS_DIR; };
+ 1CCD8F9F2117957600407114 /* avaudiocapture.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = avaudiocapture.m; path = ../../../modules/access/avaudiocapture.m; sourceTree = "<group>"; };
1CD366791B7A242E0054E39F /* VLCTimeSelectionPanelController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = VLCTimeSelectionPanelController.h; sourceTree = "<group>"; };
1CD3667A1B7A242E0054E39F /* VLCTimeSelectionPanelController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = VLCTimeSelectionPanelController.m; sourceTree = "<group>"; };
1CFE8D561EA0D3D300E94451 /* ErrorPanel.xib */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.xib; path = ErrorPanel.xib; sourceTree = "<group>"; };
@@ -1472,6 +1474,7 @@
CCDDF1B0172FF4E2007729A1 /* access */ = {
isa = PBXGroup;
children = (
+ 1CCD8F9F2117957600407114 /* avaudiocapture.m */,
DCC8017816FCA036007FE736 /* avcapture.m */,
DC1A176914E2778A001477C6 /* qtsound.m */,
);
@@ -1628,6 +1631,7 @@
1C3113961E508C6900D4DD76 /* applescript.m in Sources */,
6B8A6B0121279D2600DC29F3 /* PXSourceListTableCellView.m in Sources */,
1C3113981E508C6900D4DD76 /* VLCAudioEffectsWindowController.m in Sources */,
+ 1CCD8FA02117957700407114 /* avaudiocapture.m in Sources */,
6BBBF9851F7B257100B404CD /* VLCLogMessage.m in Sources */,
1C31139A1E508C6900D4DD76 /* VLCBookmarksWindowController.m in Sources */,
6B0AB0F01F1AC8B3003A1B4E /* VLCSlider.m in Sources */,
diff --git a/modules/MODULES_LIST b/modules/MODULES_LIST
index 79161ae3edf..01e545a8778 100644
--- a/modules/MODULES_LIST
+++ b/modules/MODULES_LIST
@@ -54,6 +54,7 @@ $Id$
* audiounit_ios: AudioUnit output plugin for iOS
* auhal: Audio output for Mac OS X based on the AUHAL API
* avahi: Zeroconf services discovery
+ * avaudiocapture: AVFoundation based audio capture module
* avcapture: AVFoundation based video capture module
* avcodec: libavcodec audio/video decoder
* avformat: libavformat demuxer
diff --git a/modules/access/Makefile.am b/modules/access/Makefile.am
index 5abe3a1a5fd..d1743dddbaa 100644
--- a/modules/access/Makefile.am
+++ b/modules/access/Makefile.am
@@ -76,6 +76,13 @@ libaccess_qtsound_plugin_la_LDFLAGS = $(AM_LDFLAGS) -rpath '$(accessdir)' -Wl,-f
access_LTLIBRARIES += $(LTLIBaccess_qtsound)
EXTRA_LTLIBRARIES += libaccess_qtsound_plugin.la
+libavaudiocapture_plugin_la_SOURCES = access/avaudiocapture.m
+libavaudiocapture_plugin_la_OBJCFLAGS = $(AM_OBJCFLAGS) -fobjc-arc
+libavaudiocapture_plugin_la_LDFLAGS = $(AM_LDFLAGS) -rpath '$(accessdir)' -Wl,-framework,Foundation -Wl,-framework,AVFoundation -Wl,-framework,CoreMedia
+if HAVE_AVFOUNDATION
+access_LTLIBRARIES += libavaudiocapture_plugin.la
+endif
+
libaccess_wasapi_plugin_la_SOURCES = access/wasapi.c
libaccess_wasapi_plugin_la_LIBADD = $(LIBCOM) -lksuser
if HAVE_WASAPI
diff --git a/modules/access/avaudiocapture.m b/modules/access/avaudiocapture.m
new file mode 100644
index 00000000000..02f2f9e1e06
--- /dev/null
+++ b/modules/access/avaudiocapture.m
@@ -0,0 +1,345 @@
+/*****************************************************************************
+ * avaudiocapture.m: AVFoundation based audio capture module
+ *****************************************************************************
+ * Copyright © 2018 VLC authors and VideoLAN
+ *
+ * Authors: Pierre d'Herbemont <pdherbemont at videolan.org>
+ * Gustaf Neumann <neumann at wu.ac.at>
+ * Michael S. Feurstein <michael.feurstein at wu.ac.at>
+ * David Fuhrmann <dfuhrmann at videolan dot org>
+ *
+ ****************************************************************************
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+/*****************************************************************************
+ * Preamble
+ *****************************************************************************/
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <assert.h>
+
+#include <vlc_common.h>
+#include <vlc_plugin.h>
+#include <vlc_input.h>
+#include <vlc_demux.h>
+#include <vlc_dialog.h>
+
+#import <AvailabilityMacros.h>
+#import <AVFoundation/AVFoundation.h>
+#import <CoreMedia/CoreMedia.h>
+
+
+#ifndef MAC_OS_X_VERSION_10_14
+ at interface AVCaptureDevice (AVCaptureDeviceAuthorizationSince10_14)
+
++ (void)requestAccessForMediaType:(AVMediaType)mediaType completionHandler:(void (^)(BOOL granted))handler API_AVAILABLE(macos(10.14), ios(7.0));
+
+ at end
+#endif
+
+/*****************************************************************************
+ * Struct
+ *****************************************************************************/
+
+typedef struct demux_sys_t
+{
+ CFTypeRef _Nullable session; // AVCaptureSession
+ es_out_id_t *p_es_audio;
+
+} demux_sys_t;
+
+
+/*****************************************************************************
+* AVFoundation Bridge
+*****************************************************************************/
+ at interface VLCAVDecompressedAudioOutput : AVCaptureAudioDataOutput<AVCaptureAudioDataOutputSampleBufferDelegate>
+{
+ demux_t *p_avcapture;
+ date_t date;
+}
+
+ at end
+
+ at implementation VLCAVDecompressedAudioOutput : AVCaptureAudioDataOutput
+
+- (id)initWithDemux:(demux_t *)p_demux
+{
+ if (self = [super init])
+ {
+ p_avcapture = p_demux;
+
+ date_Init(&date, 44100, 1);
+ date_Set(&date, VLC_TICK_0);
+ }
+ return self;
+}
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+ fromConnection:(AVCaptureConnection *)connection
+{
+ @autoreleasepool {
+ CMItemCount numSamplesInBuffer = CMSampleBufferGetNumSamples(sampleBuffer);
+
+ size_t neededBufferListSize = 0;
+ // first get needed size for buffer
+ OSStatus retValue = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, &neededBufferListSize, nil, 0, nil, nil, kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, nil);
+
+ if (retValue != noErr) {
+ msg_Err(p_avcapture, "Error getting sample list buffer size: %d", retValue);
+ return;
+ }
+
+ CMBlockBufferRef blockBuffer;
+ AudioBufferList *audioBufferList = calloc(1, neededBufferListSize);
+ retValue = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, nil, audioBufferList, neededBufferListSize, nil, nil, kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, &blockBuffer);
+
+ if (retValue != noErr) {
+ msg_Err(p_avcapture, "Cannot get samples from buffer: %d", retValue);
+ return;
+ }
+
+ if (audioBufferList->mNumberBuffers != 1) {
+ msg_Warn(p_avcapture, "This module expects 1 buffer only, got %d", audioBufferList->mNumberBuffers);
+ return;
+ }
+
+ int64_t totalDataSize = audioBufferList->mBuffers[0].mDataByteSize;
+ block_t *outBlock = block_Alloc(totalDataSize);
+ if (!outBlock)
+ return;
+
+ date_Increment(&date, (uint32_t)numSamplesInBuffer);
+ vlc_tick_t currentPts = date_Get(&date);
+
+ memcpy(outBlock->p_buffer, audioBufferList->mBuffers[0].mData, audioBufferList->mBuffers[0].mDataByteSize);
+ outBlock->i_pts = currentPts;
+ outBlock->i_nb_samples = (unsigned)numSamplesInBuffer;
+
+ CFRelease(blockBuffer);
+
+ demux_sys_t *p_sys = p_avcapture->p_sys;
+ es_out_SetPCR(p_avcapture->out, outBlock->i_pts);
+ es_out_Send(p_avcapture->out, p_sys->p_es_audio, outBlock);
+ }
+}
+
+ at end
+
+
+/*****************************************************************************
+ * Control:
+ *****************************************************************************/
+static int Control(demux_t *p_demux, int i_query, va_list args)
+{
+ bool *pb;
+
+ switch(i_query) {
+ /* Special for access_demux */
+ case DEMUX_CAN_PAUSE:
+ case DEMUX_CAN_SEEK:
+ case DEMUX_SET_PAUSE_STATE:
+ case DEMUX_CAN_CONTROL_PACE:
+ pb = (bool*)va_arg(args, bool *);
+ *pb = false;
+ return VLC_SUCCESS;
+
+ case DEMUX_GET_PTS_DELAY:
+ *va_arg(args, vlc_tick_t *) =
+ VLC_TICK_FROM_MS(var_InheritInteger(p_demux, "live-caching"));
+ return VLC_SUCCESS;
+
+ default:
+ return VLC_EGENERIC;
+ }
+ return VLC_EGENERIC;
+}
+
+/*****************************************************************************
+* Open:
+*****************************************************************************/
+static int Open(vlc_object_t *p_this)
+{
+ demux_t *p_demux = (demux_t*)p_this;
+
+ if (p_demux->out == NULL)
+ return VLC_EGENERIC;
+
+ @autoreleasepool {
+ NSString *currentDeviceId = @"";
+ if (p_demux->psz_location && *p_demux->psz_location)
+ currentDeviceId = [NSString stringWithUTF8String:p_demux->psz_location];
+
+ msg_Dbg(p_demux, "avcapture uid = %s", currentDeviceId.UTF8String);
+
+ NSArray *knownAudioDevices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
+ NSInteger numberOfKnownAudioDevices = [knownAudioDevices count];
+
+ int selectedDevice = 0;
+ for (;selectedDevice < numberOfKnownAudioDevices; selectedDevice++ )
+ {
+ AVCaptureDevice *avf_device = [knownAudioDevices objectAtIndex:selectedDevice];
+ msg_Dbg(p_demux, "avcapture %i/%ld %s %s", selectedDevice, (long)numberOfKnownAudioDevices, [[avf_device modelID] UTF8String], [[avf_device uniqueID] UTF8String]);
+ if ([[[avf_device uniqueID] stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:currentDeviceId]) {
+ break;
+ }
+ }
+
+ AVCaptureDevice *device = nil;
+ if (selectedDevice < numberOfKnownAudioDevices) {
+ device = [knownAudioDevices objectAtIndex:selectedDevice];
+ } else {
+ msg_Dbg(p_demux, "Cannot find designated device as %s, falling back to default.", currentDeviceId.UTF8String);
+ device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
+ }
+
+ if (!device) {
+ vlc_dialog_display_error(p_demux, _("No Audio Input device found"),
+ _("Your Mac does not seem to be equipped with a suitable audio input device."
+ "Please check your connectors and drivers."));
+ msg_Err(p_demux, "Can't find any Audio device");
+ return VLC_EGENERIC;
+ }
+
+ if ([device isInUseByAnotherApplication]) {
+ msg_Err(p_demux, "Capture device is exclusively in use by another application");
+ return VLC_EGENERIC;
+ }
+
+ if (@available(macOS 10.14, *)) {
+ msg_Dbg(p_demux, "Check user consent for access to the audio device");
+
+ dispatch_semaphore_t sema = dispatch_semaphore_create(0);
+ __block bool accessGranted = NO;
+ [AVCaptureDevice requestAccessForMediaType: AVMediaTypeAudio completionHandler:^(BOOL granted) {
+ accessGranted = granted;
+ dispatch_semaphore_signal(sema);
+ } ];
+ dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER);
+ if (!accessGranted) {
+ msg_Err(p_demux, "Can't use the audio device as access has not been granted by the user");
+ return VLC_EGENERIC;
+ }
+ }
+
+ NSError *error = nil;
+ AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error];
+ if (!input) {
+ msg_Err(p_demux, "can't create a valid capture input facility (%ld)", [error code]);
+ return VLC_EGENERIC;
+ }
+
+ /* Now we can init */
+ int audiocodec = VLC_CODEC_FL32;
+ es_format_t audiofmt;
+ es_format_Init(&audiofmt, AUDIO_ES, audiocodec);
+
+ audiofmt.audio.i_format = audiocodec;
+ audiofmt.audio.i_rate = 44100;
+ /*
+ * i_physical_channels Describes the channels configuration of the
+ * samples (ie. number of channels which are available in the
+ * buffer, and positions).
+ */
+ audiofmt.audio.i_physical_channels = AOUT_CHAN_RIGHT | AOUT_CHAN_LEFT;
+ /*
+ * Please note that it may be completely arbitrary - buffers are not
+ * obliged to contain a integral number of so-called "frames". It's
+ * just here for the division:
+ * buffer_size = i_nb_samples * i_bytes_per_frame / i_frame_length
+ */
+ audiofmt.audio.i_bitspersample = 32;
+ audiofmt.audio.i_channels = 2;
+ audiofmt.audio.i_blockalign = audiofmt.audio.i_channels * (audiofmt.audio.i_bitspersample / 8);
+ audiofmt.i_bitrate = audiofmt.audio.i_channels * audiofmt.audio.i_rate * audiofmt.audio.i_bitspersample;
+
+
+ AVCaptureSession *session = [[AVCaptureSession alloc] init];
+ [session addInput:input];
+
+ VLCAVDecompressedAudioOutput *output = [[VLCAVDecompressedAudioOutput alloc] initWithDemux:p_demux];
+ [session addOutput:output];
+
+ dispatch_queue_t queue = dispatch_queue_create("avCaptureQueue", NULL);
+ [output setSampleBufferDelegate:output queue:queue];
+
+ [output setAudioSettings:
+ @{
+ AVFormatIDKey : @(kAudioFormatLinearPCM),
+ AVLinearPCMBitDepthKey : @(32),
+ AVLinearPCMIsFloatKey : @YES,
+ AVLinearPCMIsBigEndianKey : @NO,
+ AVNumberOfChannelsKey : @(2),
+ AVLinearPCMIsNonInterleaved : @NO,
+ AVSampleRateKey : @(44100.0)
+ }];
+
+ /* Set up p_demux */
+ p_demux->pf_demux = NULL;
+ p_demux->pf_control = Control;
+
+ demux_sys_t *p_sys = NULL;
+ p_demux->p_sys = p_sys = calloc(1, sizeof(demux_sys_t));
+ if (!p_sys)
+ return VLC_ENOMEM;
+
+ p_sys->session = CFBridgingRetain(session);
+ p_sys->p_es_audio = es_out_Add(p_demux->out, &audiofmt);
+
+ [session startRunning];
+ msg_Dbg(p_demux, "AVCapture: Audio device ready!");
+ return VLC_SUCCESS;
+ }
+}
+
+/*****************************************************************************
+* Close:
+*****************************************************************************/
+static void Close(vlc_object_t *p_this)
+{
+ demux_t *p_demux = (demux_t*)p_this;
+ demux_sys_t *p_sys = p_demux->p_sys;
+
+ @autoreleasepool {
+ msg_Dbg(p_demux,"Close AVCapture");
+
+ ///@todo Investigate why this should be needed
+ // Perform this on main thread, as the framework itself will sometimes try to synchronously
+ // work on main thread. And this will create a dead lock.
+// [(__bridge AVCaptureSession *)p_sys->session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:YES];
+
+ [(__bridge AVCaptureSession *)p_sys->session stopRunning];
+ CFBridgingRelease(p_sys->session);
+
+ free(p_sys);
+ }
+}
+
+/*****************************************************************************
+ * Module descriptor
+ *****************************************************************************/
+vlc_module_begin ()
+set_shortname(N_("AVFoundation Audio Capture"))
+set_description(N_("AVFoundation audio capture module."))
+set_category(CAT_INPUT)
+set_subcategory(SUBCAT_INPUT_ACCESS)
+add_shortcut("qtsound")
+set_capability("access", 0)
+set_callbacks(Open, Close)
+vlc_module_end ()
diff --git a/po/POTFILES.in b/po/POTFILES.in
index 504e3ffd4bf..7908aceab11 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -163,6 +163,7 @@ lib/video.c
# modules
modules/access/alsa.c
+modules/access/avaudiocapture.m
modules/access/avcapture.m
modules/access/avio.h
modules/access/bluray.c
--
2.17.2 (Apple Git-113)
More information about the vlc-devel
mailing list