From d3b0245e8e0115f17e6670b4b0d563f98300e093 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Fri, 3 Jun 2022 08:23:45 +0200 Subject: [PATCH 01/26] note on how to fix NSCameraUsageDescription requirement --- Sources/Extensions/AVCaptureSession+BaseRecorder.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Sources/Extensions/AVCaptureSession+BaseRecorder.swift b/Sources/Extensions/AVCaptureSession+BaseRecorder.swift index 2a05621..a7030c6 100644 --- a/Sources/Extensions/AVCaptureSession+BaseRecorder.swift +++ b/Sources/Extensions/AVCaptureSession+BaseRecorder.swift @@ -23,6 +23,10 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +/* NOTE + You should exclude this class from the Target Membership if you don't use its functionality. + In this way you are not required to include NSCameraUsageDescription when pushing your build to appstoreconnect + */ import Foundation import AVFoundation From ba2c558daeb8eca771663d0bf8ac997cf60a65c7 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Fri, 3 Jun 2022 08:24:43 +0200 Subject: [PATCH 02/26] fix wrong color profile while recording on some devices --- Sources/MediaSession/MediaSession.swift | 2 +- Sources/VideoSettings.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/MediaSession/MediaSession.swift b/Sources/MediaSession/MediaSession.swift index b1c025a..cfc808c 100644 --- a/Sources/MediaSession/MediaSession.swift +++ b/Sources/MediaSession/MediaSession.swift @@ -148,7 +148,7 @@ extension MediaSession { var videoSettings = videoSettings if videoSettings.size == nil { videoSettings.size = videoInput.size } if videoSettings.transform == nil { videoSettings.transform = videoInput.videoTransform } - videoSettings.videoColorProperties = videoInput.videoColorProperties + //videoSettings.videoColorProperties = videoInput.videoColorProperties let audioSettingsDictionary = audioSettings?.outputSettings ?? audioInput?.recommendedAudioSettingsForAssetWriter( diff --git a/Sources/VideoSettings.swift b/Sources/VideoSettings.swift index 2b7af92..017fd7c 100644 --- a/Sources/VideoSettings.swift +++ b/Sources/VideoSettings.swift @@ -73,7 +73,7 @@ extension VideoSettings { AVVideoHeightKey: size?.height, AVVideoCodecKey: codec.avCodec, AVVideoScalingModeKey: scalingMode.avScalingMode, - AVVideoColorPropertiesKey: videoColorProperties, + //AVVideoColorPropertiesKey: videoColorProperties, AVVideoCompressionPropertiesKey: codec.compressionProperties ] as [String: Any?]).compactMapValues({ $0 }) } From 0404b2232060773189b22f79c63a6f9d154b041f Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Fri, 3 Jun 2022 09:28:18 +0200 Subject: [PATCH 03/26] move description of NSCameraUsageDescription fix to README --- README.md | 2 ++ Sources/Extensions/AVCaptureSession+BaseRecorder.swift | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2a3c4b2..bd38591 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,8 @@ override func viewDidLoad() { } ``` + Knows issues: Capturing audio requires that you provide a valid NSCameraUsageDescription (or NSMicrophoneUsageDescription?) key in the InfoPlist when you submit your app to appstoreconnect. Anyway, If you are not using the audio recorder, you can exclude the `AVCaptureSession+BaseRecorder` extension from the Target Membership and this will fix appstoreconnect asking for NSCameraUsageDescription key. + ### Music Overlay Instead of capturing audio using microphone you can play music and add it to video at the same time. diff --git a/Sources/Extensions/AVCaptureSession+BaseRecorder.swift b/Sources/Extensions/AVCaptureSession+BaseRecorder.swift index a7030c6..2a05621 100644 --- a/Sources/Extensions/AVCaptureSession+BaseRecorder.swift +++ b/Sources/Extensions/AVCaptureSession+BaseRecorder.swift @@ -23,10 +23,6 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -/* NOTE - You should exclude this class from the Target Membership if you don't use its functionality. - In this way you are not required to include NSCameraUsageDescription when pushing your build to appstoreconnect - */ import Foundation import AVFoundation From 5e81cf92757df16c376e12607bb93f0b2aeac75e Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Fri, 3 Jun 2022 10:34:34 +0200 Subject: [PATCH 04/26] modify instructions on how to fix NSCameraUsageDescription --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bd38591..91c814c 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ override func viewDidLoad() { } ``` - Knows issues: Capturing audio requires that you provide a valid NSCameraUsageDescription (or NSMicrophoneUsageDescription?) key in the InfoPlist when you submit your app to appstoreconnect. Anyway, If you are not using the audio recorder, you can exclude the `AVCaptureSession+BaseRecorder` extension from the Target Membership and this will fix appstoreconnect asking for NSCameraUsageDescription key. + Knows issues: Capturing audio requires that you provide a valid NSCameraUsageDescription key in the InfoPlist when you submit your app to appstoreconnect. If you are not using the audio recorder, you can exclude the `AVCaptureSession+BaseRecorder.swift` extension from the Target Membership. Doing that fixes appstoreconnect asking for NSCameraUsageDescription key. ### Music Overlay From 5de693bd9dca07a5453efcf006dfe3238141d76f Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Fri, 3 Jun 2022 10:37:10 +0200 Subject: [PATCH 05/26] fix typo and bold for known issue: --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 91c814c..43916b9 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ override func viewDidLoad() { } ``` - Knows issues: Capturing audio requires that you provide a valid NSCameraUsageDescription key in the InfoPlist when you submit your app to appstoreconnect. If you are not using the audio recorder, you can exclude the `AVCaptureSession+BaseRecorder.swift` extension from the Target Membership. Doing that fixes appstoreconnect asking for NSCameraUsageDescription key. + **Known issue:** Capturing audio requires that you provide a valid NSCameraUsageDescription key in the InfoPlist when you submit your app to appstoreconnect. If you are not using the audio recorder, you can exclude the `AVCaptureSession+BaseRecorder.swift` extension from the Target Membership. Doing that fixes appstoreconnect asking for NSCameraUsageDescription key. ### Music Overlay From c1ed9603f4b801870bb7d953a8a52b8c0170414e Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Tue, 7 Jun 2022 10:00:43 +0200 Subject: [PATCH 06/26] change priority of queue to userInteractive to solve lag problem on slower devices --- Sources/AudioEngine/AudioEngine.swift | 2 +- Sources/Recorder/CleanRecorder/CleanRecorder.swift | 2 +- Sources/Recorder/SceneRecorder/SceneRecorder.swift | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/AudioEngine/AudioEngine.swift b/Sources/AudioEngine/AudioEngine.swift index adbcc53..61d9a04 100644 --- a/Sources/AudioEngine/AudioEngine.swift +++ b/Sources/AudioEngine/AudioEngine.swift @@ -47,7 +47,7 @@ public final class AudioEngine { } } - let queue = DispatchQueue(label: "AudioEngine.Processing", qos: .userInitiated) + let queue = DispatchQueue(label: "AudioEngine.Processing", qos: .userInteractive) lazy var engine: AVAudioEngine = { let engine = AVAudioEngine() diff --git a/Sources/Recorder/CleanRecorder/CleanRecorder.swift b/Sources/Recorder/CleanRecorder/CleanRecorder.swift index 467567a..134cb9e 100644 --- a/Sources/Recorder/CleanRecorder/CleanRecorder.swift +++ b/Sources/Recorder/CleanRecorder/CleanRecorder.swift @@ -32,7 +32,7 @@ public final class CleanRecorder: BaseRecorder, let videoInput: VideoInput init(_ cleanRecordable: T, timeScale: CMTimeScale = 600) { - let queue = DispatchQueue(label: "SCNRecorder.Processing.DispatchQueue", qos: .userInitiated) + let queue = DispatchQueue(label: "SCNRecorder.Processing.DispatchQueue", qos: .userInteractive) self.videoInput = VideoInput( cleanRecordable: cleanRecordable, diff --git a/Sources/Recorder/SceneRecorder/SceneRecorder.swift b/Sources/Recorder/SceneRecorder/SceneRecorder.swift index c96989c..87edb15 100644 --- a/Sources/Recorder/SceneRecorder/SceneRecorder.swift +++ b/Sources/Recorder/SceneRecorder/SceneRecorder.swift @@ -45,7 +45,7 @@ public final class SceneRecorder: BaseRecorder, Renderable, SCNSceneRendererDele ) throws { let queue = DispatchQueue( label: "SCNRecorder.Processing.DispatchQueue", - qos: .userInitiated + qos: .userInteractive ) try self.init( videoInput: VideoInput( From 963e935688f1b94a4c136bfbf1609cc78ea5c401 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Tue, 26 Jul 2022 14:11:11 +0200 Subject: [PATCH 07/26] revert disable videoColorProperties --- Sources/MediaSession/MediaSession.swift | 2 +- Sources/VideoSettings.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/MediaSession/MediaSession.swift b/Sources/MediaSession/MediaSession.swift index cfc808c..b1c025a 100644 --- a/Sources/MediaSession/MediaSession.swift +++ b/Sources/MediaSession/MediaSession.swift @@ -148,7 +148,7 @@ extension MediaSession { var videoSettings = videoSettings if videoSettings.size == nil { videoSettings.size = videoInput.size } if videoSettings.transform == nil { videoSettings.transform = videoInput.videoTransform } - //videoSettings.videoColorProperties = videoInput.videoColorProperties + videoSettings.videoColorProperties = videoInput.videoColorProperties let audioSettingsDictionary = audioSettings?.outputSettings ?? audioInput?.recommendedAudioSettingsForAssetWriter( diff --git a/Sources/VideoSettings.swift b/Sources/VideoSettings.swift index 017fd7c..2b7af92 100644 --- a/Sources/VideoSettings.swift +++ b/Sources/VideoSettings.swift @@ -73,7 +73,7 @@ extension VideoSettings { AVVideoHeightKey: size?.height, AVVideoCodecKey: codec.avCodec, AVVideoScalingModeKey: scalingMode.avScalingMode, - //AVVideoColorPropertiesKey: videoColorProperties, + AVVideoColorPropertiesKey: videoColorProperties, AVVideoCompressionPropertiesKey: codec.compressionProperties ] as [String: Any?]).compactMapValues({ $0 }) } From e63b55a589df97e092b54952cb96e73feb622a3f Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Wed, 27 Jul 2022 13:44:38 +0200 Subject: [PATCH 08/26] disable videoColorProperties for iOS 12 and lower --- Sources/MediaSession/MediaSession.swift | 4 +++- Sources/VideoSettings.swift | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Sources/MediaSession/MediaSession.swift b/Sources/MediaSession/MediaSession.swift index b1c025a..bcd1982 100644 --- a/Sources/MediaSession/MediaSession.swift +++ b/Sources/MediaSession/MediaSession.swift @@ -148,7 +148,9 @@ extension MediaSession { var videoSettings = videoSettings if videoSettings.size == nil { videoSettings.size = videoInput.size } if videoSettings.transform == nil { videoSettings.transform = videoInput.videoTransform } - videoSettings.videoColorProperties = videoInput.videoColorProperties + if #available(iOS 13, *) { + videoSettings.videoColorProperties = videoInput.videoColorProperties + } let audioSettingsDictionary = audioSettings?.outputSettings ?? audioInput?.recommendedAudioSettingsForAssetWriter( diff --git a/Sources/VideoSettings.swift b/Sources/VideoSettings.swift index 2b7af92..efb8a96 100644 --- a/Sources/VideoSettings.swift +++ b/Sources/VideoSettings.swift @@ -68,13 +68,17 @@ public struct VideoSettings { extension VideoSettings { var outputSettings: [String: Any] { - return ([ - AVVideoWidthKey: size?.width, - AVVideoHeightKey: size?.height, - AVVideoCodecKey: codec.avCodec, - AVVideoScalingModeKey: scalingMode.avScalingMode, - AVVideoColorPropertiesKey: videoColorProperties, - AVVideoCompressionPropertiesKey: codec.compressionProperties - ] as [String: Any?]).compactMapValues({ $0 }) + var ostgs : [String: Any?] = [ + AVVideoWidthKey: size?.width, + AVVideoHeightKey: size?.height, + AVVideoCodecKey: codec.avCodec, + AVVideoScalingModeKey: scalingMode.avScalingMode, + AVVideoColorPropertiesKey: videoColorProperties, + AVVideoCompressionPropertiesKey: codec.compressionProperties + ] + if #available(iOS 13, *) { + ostgs[AVVideoColorPropertiesKey]=videoColorProperties + } + return ostgs.compactMapValues({ $0 }) } } From f21c22517c16f9c78382ac08cf630c62e5f58f70 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Tue, 27 Jun 2023 14:13:39 +0200 Subject: [PATCH 09/26] reverse changes from original repository --- Sources/AudioEngine/AudioEngine.swift | 1 - Sources/MediaSession/MediaSession.swift | 4 +--- .../CleanRecorder/CleanRecorder.swift | 2 +- .../SceneRecorder/SceneRecorder.swift | 2 +- Sources/VideoSettings.swift | 20 ++++++++----------- 5 files changed, 11 insertions(+), 18 deletions(-) diff --git a/Sources/AudioEngine/AudioEngine.swift b/Sources/AudioEngine/AudioEngine.swift index 61d9a04..49f918d 100644 --- a/Sources/AudioEngine/AudioEngine.swift +++ b/Sources/AudioEngine/AudioEngine.swift @@ -47,7 +47,6 @@ public final class AudioEngine { } } - let queue = DispatchQueue(label: "AudioEngine.Processing", qos: .userInteractive) lazy var engine: AVAudioEngine = { let engine = AVAudioEngine() diff --git a/Sources/MediaSession/MediaSession.swift b/Sources/MediaSession/MediaSession.swift index 3ac0650..ca235ed 100644 --- a/Sources/MediaSession/MediaSession.swift +++ b/Sources/MediaSession/MediaSession.swift @@ -145,9 +145,7 @@ extension MediaSession { var videoSettings = videoSettings if videoSettings.size == nil { videoSettings.size = videoInput.size } if videoSettings.transform == nil { videoSettings.transform = videoInput.videoTransform } - if #available(iOS 13, *) { - videoSettings.videoColorProperties = videoInput.videoColorProperties - } + videoSettings.videoColorProperties = videoInput.videoColorProperties let audioSettingsDictionary = audioSettings?.outputSettings ?? audioInput?.recommendedAudioSettingsForAssetWriter( diff --git a/Sources/Recorder/CleanRecorder/CleanRecorder.swift b/Sources/Recorder/CleanRecorder/CleanRecorder.swift index aaf5b1f..158476f 100644 --- a/Sources/Recorder/CleanRecorder/CleanRecorder.swift +++ b/Sources/Recorder/CleanRecorder/CleanRecorder.swift @@ -33,7 +33,7 @@ public final class CleanRecorder: BaseRecorder, let videoInput: VideoInput init(_ cleanRecordable: T, timeScale: CMTimeScale = 600) { - let queue = DispatchQueue(label: "SCNRecorder.Processing.DispatchQueue", qos: .userInteractive) + let queue = DispatchQueue(label: "SCNRecorder.Processing.DispatchQueue", qos: .userInitiated) self.videoInput = VideoInput( cleanRecordable: cleanRecordable, diff --git a/Sources/Recorder/SceneRecorder/SceneRecorder.swift b/Sources/Recorder/SceneRecorder/SceneRecorder.swift index 2f0788c..f2dd5b5 100644 --- a/Sources/Recorder/SceneRecorder/SceneRecorder.swift +++ b/Sources/Recorder/SceneRecorder/SceneRecorder.swift @@ -45,7 +45,7 @@ public final class SceneRecorder: BaseRecorder, Renderable, SCNSceneRendererDele ) throws { let queue = DispatchQueue( label: "SCNRecorder.Processing.DispatchQueue", - qos: .userInteractive + qos: .userInitiated ) try self.init( videoInput: VideoInput( diff --git a/Sources/VideoSettings.swift b/Sources/VideoSettings.swift index efb8a96..2b7af92 100644 --- a/Sources/VideoSettings.swift +++ b/Sources/VideoSettings.swift @@ -68,17 +68,13 @@ public struct VideoSettings { extension VideoSettings { var outputSettings: [String: Any] { - var ostgs : [String: Any?] = [ - AVVideoWidthKey: size?.width, - AVVideoHeightKey: size?.height, - AVVideoCodecKey: codec.avCodec, - AVVideoScalingModeKey: scalingMode.avScalingMode, - AVVideoColorPropertiesKey: videoColorProperties, - AVVideoCompressionPropertiesKey: codec.compressionProperties - ] - if #available(iOS 13, *) { - ostgs[AVVideoColorPropertiesKey]=videoColorProperties - } - return ostgs.compactMapValues({ $0 }) + return ([ + AVVideoWidthKey: size?.width, + AVVideoHeightKey: size?.height, + AVVideoCodecKey: codec.avCodec, + AVVideoScalingModeKey: scalingMode.avScalingMode, + AVVideoColorPropertiesKey: videoColorProperties, + AVVideoCompressionPropertiesKey: codec.compressionProperties + ] as [String: Any?]).compactMapValues({ $0 }) } } From 43c346eab7c0d570795e42b170d11b255d71f814 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Tue, 27 Jun 2023 14:18:11 +0200 Subject: [PATCH 10/26] restore wrongly deleted line --- Sources/AudioEngine/AudioEngine.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/AudioEngine/AudioEngine.swift b/Sources/AudioEngine/AudioEngine.swift index 49f918d..adbcc53 100644 --- a/Sources/AudioEngine/AudioEngine.swift +++ b/Sources/AudioEngine/AudioEngine.swift @@ -47,6 +47,7 @@ public final class AudioEngine { } } + let queue = DispatchQueue(label: "AudioEngine.Processing", qos: .userInitiated) lazy var engine: AVAudioEngine = { let engine = AVAudioEngine() From 0eab6409cf63dba7f84a27e7b8a959551a767965 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Fri, 8 Dec 2023 12:19:00 +0100 Subject: [PATCH 11/26] align branch with stable v2.8.1 --- .../VideoOutput/VideoOutput.State.swift | 55 +++++++------------ Sources/Outputs/VideoOutput/VideoOutput.swift | 20 ++----- Sources/SelfRecordable/SelfRecordable.swift | 8 --- 3 files changed, 26 insertions(+), 57 deletions(-) diff --git a/Sources/Outputs/VideoOutput/VideoOutput.State.swift b/Sources/Outputs/VideoOutput/VideoOutput.State.swift index a6a6942..4a65fd5 100644 --- a/Sources/Outputs/VideoOutput/VideoOutput.State.swift +++ b/Sources/Outputs/VideoOutput/VideoOutput.State.swift @@ -61,10 +61,12 @@ public enum VideoOutputState: Equatable { /// Recording is active. /// Appending audio and video frames. - case recording(time: CMTime, pause: CMTime?, resume: CMTime?) + case recording(time: CMTime) /// Recording is paused. - case paused(time: CMTime, pause: CMTime) + /// Not tested. + /// According to Apple, documentation might not work. + case paused /// Recording is canceled. /// Final state, not further actions are possible. @@ -105,12 +107,11 @@ public enum VideoOutputState: Equatable { .preparing: return .preparing - case let .recording(time, pause, resume): - return .recording(time: time, pause: pause, resume: resume) + case .recording(let time): + return .recording(time: time) - case let .paused(time, pause): - let resume = CMTime(seconds: CACurrentMediaTime(), preferredTimescale: pause.timescale) - return .recording(time: time, pause: pause, resume: resume) + case .paused: + return .preparing case .canceled, .finished, @@ -129,14 +130,9 @@ public enum VideoOutputState: Equatable { .preparing: return .ready - case let .recording(lastTime, pause, resume): - let current = CMTime(seconds: CACurrentMediaTime(), preferredTimescale: lastTime.timescale) - if let pause, let resume { - let adjustedPause = adjustTime(current: current, resume: resume, pause: pause) - return .paused(time: lastTime, pause: adjustedPause) - } else { - return .paused(time: lastTime, pause: current) - } + case .recording(let seconds): + videoOutput.endSession(at: seconds) + return .paused case .paused, .canceled, @@ -233,13 +229,14 @@ private extension VideoOutputState { } else { time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) } + videoOutput.startSession(at: time) try videoOutput.appendVideo(sampleBuffer: sampleBuffer) - return .recording(time: time, pause: nil, resume: nil) + return .recording(time: time) - case let .recording(time, pause, resume): + case .recording(let time): try videoOutput.appendVideo(sampleBuffer: sampleBuffer) - return .recording(time: time, pause: pause, resume: resume) + return .recording(time: time) case .paused, .canceled, @@ -263,17 +260,11 @@ private extension VideoOutputState { case .preparing: videoOutput.startSession(at: time) try videoOutput.append(pixelBuffer: pixelBuffer, withPresentationTime: time) - return .recording(time: time, pause: nil, resume: nil) + return .recording(time: time) - case let .recording(_ ,pause, resume): - let finalTime: CMTime - if let pause, let resume { - finalTime = adjustTime(current: time, resume: resume, pause: pause) - } else { - finalTime = time - } - try videoOutput.append(pixelBuffer: pixelBuffer, withPresentationTime: finalTime) - return .recording(time: finalTime, pause: pause, resume: resume) + case .recording: + try videoOutput.append(pixelBuffer: pixelBuffer, withPresentationTime: time) + return .recording(time: time) case .paused, .canceled, @@ -294,9 +285,9 @@ private extension VideoOutputState { .preparing: return self - case let .recording(time, pause, resume): + case .recording(let time): try videoOutput.appendAudio(sampleBuffer: sampleBuffer) - return .recording(time: time, pause: pause, resume: resume) + return .recording(time: time) case .paused, .canceled, @@ -305,8 +296,4 @@ private extension VideoOutputState { return self } } - - func adjustTime(current: CMTime, resume: CMTime, pause: CMTime) -> CMTime { - return CMTimeSubtract(current, CMTimeSubtract(resume, pause)) - } } diff --git a/Sources/Outputs/VideoOutput/VideoOutput.swift b/Sources/Outputs/VideoOutput/VideoOutput.swift index 131cbc9..06c7c69 100644 --- a/Sources/Outputs/VideoOutput/VideoOutput.swift +++ b/Sources/Outputs/VideoOutput/VideoOutput.swift @@ -141,9 +141,7 @@ extension VideoOutput { } func append(pixelBuffer: CVPixelBuffer, withPresentationTime time: CMTime) throws { - guard pixelBufferAdaptor.assetWriterInput.isReadyForMoreMediaData else { - return - } + guard pixelBufferAdaptor.assetWriterInput.isReadyForMoreMediaData else { return } guard pixelBufferAdaptor.append(pixelBuffer, withPresentationTime: time) else { if assetWriter.status == .failed { throw assetWriter.error ?? Error.unknown } return @@ -155,9 +153,7 @@ extension VideoOutput { } func appendVideo(sampleBuffer: CMSampleBuffer) throws { - guard videoInput.isReadyForMoreMediaData else { - return - } + guard videoInput.isReadyForMoreMediaData else { return } guard videoInput.append(sampleBuffer) else { if assetWriter.status == .failed { throw assetWriter.error ?? Error.unknown } return @@ -179,16 +175,10 @@ extension VideoOutput { } func appendAudio(sampleBuffer: CMSampleBuffer) throws { - guard - let audioInput = audioInput, - audioInput.isReadyForMoreMediaData - else { - return - } + guard let audioInput = audioInput else { return } + guard audioInput.isReadyForMoreMediaData else { return } guard audioInput.append(sampleBuffer) else { - if assetWriter.status == .failed { - throw assetWriter.error ?? Error.unknown - } + if assetWriter.status == .failed { throw assetWriter.error ?? Error.unknown } return } } diff --git a/Sources/SelfRecordable/SelfRecordable.swift b/Sources/SelfRecordable/SelfRecordable.swift index f55e270..5838815 100644 --- a/Sources/SelfRecordable/SelfRecordable.swift +++ b/Sources/SelfRecordable/SelfRecordable.swift @@ -189,14 +189,6 @@ public extension SelfRecordable { return videoRecording } - func pauseVideoRecording() { - videoRecording?.pause() - } - - func resumeVideoRecording() { - videoRecording?.resume() - } - func finishVideoRecording(completionHandler handler: @escaping (VideoRecording.Info) -> Void) { videoRecording?.finish { videoRecordingInfo in DispatchQueue.main.async { handler(videoRecordingInfo) } From 81d1d64a87b7211dac648358977499fb9c2b4d69 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Mon, 11 Dec 2023 15:58:08 +0100 Subject: [PATCH 12/26] Increase priority of queues to userInteractive. --- Sources/AudioEngine/AudioEngine.swift | 2 +- Sources/Recorder/CleanRecorder/CleanRecorder.swift | 2 +- Sources/Recorder/SceneRecorder/SceneRecorder.swift | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/AudioEngine/AudioEngine.swift b/Sources/AudioEngine/AudioEngine.swift index adbcc53..61d9a04 100644 --- a/Sources/AudioEngine/AudioEngine.swift +++ b/Sources/AudioEngine/AudioEngine.swift @@ -47,7 +47,7 @@ public final class AudioEngine { } } - let queue = DispatchQueue(label: "AudioEngine.Processing", qos: .userInitiated) + let queue = DispatchQueue(label: "AudioEngine.Processing", qos: .userInteractive) lazy var engine: AVAudioEngine = { let engine = AVAudioEngine() diff --git a/Sources/Recorder/CleanRecorder/CleanRecorder.swift b/Sources/Recorder/CleanRecorder/CleanRecorder.swift index 158476f..aaf5b1f 100644 --- a/Sources/Recorder/CleanRecorder/CleanRecorder.swift +++ b/Sources/Recorder/CleanRecorder/CleanRecorder.swift @@ -33,7 +33,7 @@ public final class CleanRecorder: BaseRecorder, let videoInput: VideoInput init(_ cleanRecordable: T, timeScale: CMTimeScale = 600) { - let queue = DispatchQueue(label: "SCNRecorder.Processing.DispatchQueue", qos: .userInitiated) + let queue = DispatchQueue(label: "SCNRecorder.Processing.DispatchQueue", qos: .userInteractive) self.videoInput = VideoInput( cleanRecordable: cleanRecordable, diff --git a/Sources/Recorder/SceneRecorder/SceneRecorder.swift b/Sources/Recorder/SceneRecorder/SceneRecorder.swift index f2dd5b5..2f0788c 100644 --- a/Sources/Recorder/SceneRecorder/SceneRecorder.swift +++ b/Sources/Recorder/SceneRecorder/SceneRecorder.swift @@ -45,7 +45,7 @@ public final class SceneRecorder: BaseRecorder, Renderable, SCNSceneRendererDele ) throws { let queue = DispatchQueue( label: "SCNRecorder.Processing.DispatchQueue", - qos: .userInitiated + qos: .userInteractive ) try self.init( videoInput: VideoInput( From 7215150fb8f0daca09dc771a8bda9a87756cc252 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Thu, 21 Dec 2023 12:11:19 +0100 Subject: [PATCH 13/26] Add option to include metadata in VideoSettings --- Sources/Outputs/VideoOutput/VideoOutput.swift | 4 +++- Sources/VideoSettings.swift | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Sources/Outputs/VideoOutput/VideoOutput.swift b/Sources/Outputs/VideoOutput/VideoOutput.swift index 06c7c69..23d57f6 100644 --- a/Sources/Outputs/VideoOutput/VideoOutput.swift +++ b/Sources/Outputs/VideoOutput/VideoOutput.swift @@ -91,7 +91,9 @@ final class VideoOutput { let videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings.outputSettings) videoInput.expectsMediaDataInRealTime = true videoInput.transform = videoSettings.transform ?? .identity - + if let metadata = videoSettings.metadata { + videoInput.metadata = metadata + } guard assetWriter.canAdd(videoInput) else { throw Error.cantAddVideoAssetWriterInput } assetWriter.add(videoInput) self.videoInput = videoInput diff --git a/Sources/VideoSettings.swift b/Sources/VideoSettings.swift index 2b7af92..8b2e0d0 100644 --- a/Sources/VideoSettings.swift +++ b/Sources/VideoSettings.swift @@ -48,6 +48,10 @@ public struct VideoSettings { /// Be carefull, the value is not always obvious. public var transform: CGAffineTransform? + + /// The metadata of the video + public var metadata : [AVMetadataItem]? + var videoColorProperties: [String: String]? public init( @@ -55,13 +59,15 @@ public struct VideoSettings { codec: Codec = .h264(), size: CGSize? = nil, scalingMode: ScalingMode = .resizeAspectFill, - transform: CGAffineTransform? = nil + transform: CGAffineTransform? = nil, + metadata: [AVMetadataItem]? = nil ) { self.fileType = fileType self.codec = codec self.size = size self.scalingMode = scalingMode self.transform = transform + self.metadata = metadata } } From ee15d882c029cd8fc84a43038a60824e71c894e7 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Tue, 17 Sep 2024 16:34:03 +0200 Subject: [PATCH 14/26] Merge commit by polycamnick on original repo named `obs->scnobs name conflict resolution`. Solve issue with iOS 18 compilation failure. --- Sources/AudioEngine/AudioEngine.Player.swift | 4 ++-- Sources/AudioEngine/AudioEngine.swift | 2 +- Sources/Helpers/Multithreading/Atomic.swift | 2 +- .../Multithreading/Dispatch/DispatchAtomic.swift | 2 +- .../Multithreading/ReadWrite/ReadWriteAtomic.swift | 2 +- .../Helpers/Multithreading/Unfair/UnfairAtomic.swift | 2 +- Sources/Helpers/Observable.swift | 10 +++++----- Sources/MediaSession/MediaSession.swift | 2 +- Sources/Recorder/BaseRecorder/BaseRecorder.swift | 2 +- Sources/VideoRecording.swift | 4 ++-- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Sources/AudioEngine/AudioEngine.Player.swift b/Sources/AudioEngine/AudioEngine.Player.swift index 903762a..8ffe62c 100644 --- a/Sources/AudioEngine/AudioEngine.Player.swift +++ b/Sources/AudioEngine/AudioEngine.Player.swift @@ -61,9 +61,9 @@ extension AudioEngine { }() /// Current player time. Main queue. Unique values. - @Observable public private(set) var time: TimeInterval = 0.0 + @SCNObservable public private(set) var time: TimeInterval = 0.0 - @Observable public private(set) var state = State.stopped { + @SCNObservable public private(set) var state = State.stopped { didSet { guard state != oldValue else { return } updater.isPaused = !state.isPlaying diff --git a/Sources/AudioEngine/AudioEngine.swift b/Sources/AudioEngine/AudioEngine.swift index 61d9a04..2194d85 100644 --- a/Sources/AudioEngine/AudioEngine.swift +++ b/Sources/AudioEngine/AudioEngine.swift @@ -158,7 +158,7 @@ public final class AudioEngine { let canDeactivateAudioSession: Bool - @Observable public internal(set) var error: Swift.Error? + @SCNObservable public internal(set) var error: Swift.Error? public init(canDeactivateAudioSession: Bool = true) { self.canDeactivateAudioSession = canDeactivateAudioSession diff --git a/Sources/Helpers/Multithreading/Atomic.swift b/Sources/Helpers/Multithreading/Atomic.swift index a3b3080..a4039b0 100644 --- a/Sources/Helpers/Multithreading/Atomic.swift +++ b/Sources/Helpers/Multithreading/Atomic.swift @@ -53,7 +53,7 @@ public extension Atomic { } } -public extension Atomic where Value: ObservableInterface { +public extension Atomic where Value: SCNObservableInterface { var observer: Value.Observer? { get { value.observer } diff --git a/Sources/Helpers/Multithreading/Dispatch/DispatchAtomic.swift b/Sources/Helpers/Multithreading/Dispatch/DispatchAtomic.swift index a90a0be..9d58cf0 100644 --- a/Sources/Helpers/Multithreading/Dispatch/DispatchAtomic.swift +++ b/Sources/Helpers/Multithreading/Dispatch/DispatchAtomic.swift @@ -64,4 +64,4 @@ public final class DispatchAtomic: Atomic { } } -extension DispatchAtomic: ObservableInterface where Value: ObservableInterface { } +extension DispatchAtomic: SCNObservableInterface where Value: SCNObservableInterface { } diff --git a/Sources/Helpers/Multithreading/ReadWrite/ReadWriteAtomic.swift b/Sources/Helpers/Multithreading/ReadWrite/ReadWriteAtomic.swift index 526421a..d5954b3 100644 --- a/Sources/Helpers/Multithreading/ReadWrite/ReadWriteAtomic.swift +++ b/Sources/Helpers/Multithreading/ReadWrite/ReadWriteAtomic.swift @@ -52,4 +52,4 @@ public final class ReadWriteAtomic: Atomic { } } -extension ReadWriteAtomic: ObservableInterface where Value: ObservableInterface { } +extension ReadWriteAtomic: SCNObservableInterface where Value: SCNObservableInterface { } diff --git a/Sources/Helpers/Multithreading/Unfair/UnfairAtomic.swift b/Sources/Helpers/Multithreading/Unfair/UnfairAtomic.swift index 7237cbd..8061234 100644 --- a/Sources/Helpers/Multithreading/Unfair/UnfairAtomic.swift +++ b/Sources/Helpers/Multithreading/Unfair/UnfairAtomic.swift @@ -52,4 +52,4 @@ public final class UnfairAtomic: Atomic { } } -extension UnfairAtomic: ObservableInterface where Value: ObservableInterface { } +extension UnfairAtomic: SCNObservableInterface where Value: SCNObservableInterface { } diff --git a/Sources/Helpers/Observable.swift b/Sources/Helpers/Observable.swift index 881c6a6..8f4b5a7 100644 --- a/Sources/Helpers/Observable.swift +++ b/Sources/Helpers/Observable.swift @@ -25,7 +25,7 @@ import Foundation -public protocol ObservableInterface: AnyObject { +public protocol SCNObservableInterface: AnyObject { associatedtype Property @@ -35,7 +35,7 @@ public protocol ObservableInterface: AnyObject { } @propertyWrapper -public final class Observable: ObservableInterface { +public final class SCNObservable: SCNObservableInterface { public typealias Observer = (Property) -> Void @@ -45,14 +45,14 @@ public final class Observable: ObservableInterface { public var value: Property { wrappedValue } - public var projectedValue: Observable { self } + public var projectedValue: SCNObservable { self } public var observer: Observer? public init(wrappedValue: Property) { self.wrappedValue = wrappedValue } } -public extension ObservableInterface { +public extension SCNObservableInterface { func observe( on queue: DispatchQueue? = nil, @@ -66,7 +66,7 @@ public extension ObservableInterface { } } -public extension ObservableInterface where Property: Equatable { +public extension SCNObservableInterface where Property: Equatable { func observeUnique( on queue: DispatchQueue? = nil, diff --git a/Sources/MediaSession/MediaSession.swift b/Sources/MediaSession/MediaSession.swift index ca235ed..6f75e32 100644 --- a/Sources/MediaSession/MediaSession.swift +++ b/Sources/MediaSession/MediaSession.swift @@ -62,7 +62,7 @@ final class MediaSession { @UnfairAtomic var audioOutputs = [AudioMediaSessionOutput]() - @Observable var error: Swift.Error? + @SCNObservable var error: Swift.Error? let videoInput: VideoInput diff --git a/Sources/Recorder/BaseRecorder/BaseRecorder.swift b/Sources/Recorder/BaseRecorder/BaseRecorder.swift index c0c91ab..7fcb541 100644 --- a/Sources/Recorder/BaseRecorder/BaseRecorder.swift +++ b/Sources/Recorder/BaseRecorder/BaseRecorder.swift @@ -42,7 +42,7 @@ public class BaseRecorder: NSObject { let queue: DispatchQueue - @Observable public internal(set) var error: Swift.Error? + @SCNObservable public internal(set) var error: Swift.Error? public var useAudioEngine: Bool { get { audioInput.useAudioEngine } diff --git a/Sources/VideoRecording.swift b/Sources/VideoRecording.swift index 64aa6ae..4616f8c 100644 --- a/Sources/VideoRecording.swift +++ b/Sources/VideoRecording.swift @@ -28,9 +28,9 @@ import AVFoundation public final class VideoRecording { - @Observable public internal(set) var duration: TimeInterval + @SCNObservable public internal(set) var duration: TimeInterval - @Observable public internal(set) var state: State + @SCNObservable public internal(set) var state: State public var url: URL { videoOutput.url } From c5bd1520400cde48ae8790bae6fd721efe78bd61 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Wed, 20 Nov 2024 12:48:00 +0100 Subject: [PATCH 15/26] Audio is recording! But audio can't be heard while recording. --- Sources/AudioEngine/AudioEngine.swift | 2 +- .../AudioEngine/AudioEngineFromVideo.swift | 175 ++++++++++++++++++ .../BaseRecorder.AudioInput.swift | 10 +- 3 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 Sources/AudioEngine/AudioEngineFromVideo.swift diff --git a/Sources/AudioEngine/AudioEngine.swift b/Sources/AudioEngine/AudioEngine.swift index 2194d85..eb28d8e 100644 --- a/Sources/AudioEngine/AudioEngine.swift +++ b/Sources/AudioEngine/AudioEngine.swift @@ -147,7 +147,7 @@ public final class AudioEngine { do { let sampleBuffer = try Self.createAudioSampleBuffer(from: buffer, time: time) - recorder.audioInput.audioEngine(self, didOutputAudioSampleBuffer: sampleBuffer) + recorder.audioInput.audioEngine(didOutputAudioSampleBuffer: sampleBuffer) } catch { self.error = error diff --git a/Sources/AudioEngine/AudioEngineFromVideo.swift b/Sources/AudioEngine/AudioEngineFromVideo.swift new file mode 100644 index 0000000..43d2481 --- /dev/null +++ b/Sources/AudioEngine/AudioEngineFromVideo.swift @@ -0,0 +1,175 @@ +import Foundation +import AVFoundation + +public final class AudioEngineFromVideo { + private var player: AVPlayer + private var playerItem: AVPlayerItem + private var audioFormat: AVAudioFormat? = AVAudioFormat(standardFormatWithSampleRate: 44100, channels: 2) + + @SCNObservable public internal(set) var error: Swift.Error? + + public weak var recorder: BaseRecorder? { + didSet { + oldValue?.audioInput.audioFormat = nil + guard let recorder = recorder else { + removeAudioTap() + return + } + + recorder.audioInput.audioFormat = audioFormat + + guard oldValue == nil else { return } + removeAudioTap() + setupAudioTap() + } + } + + public init(player: AVPlayer) { + self.player = player + guard let currentItem = player.currentItem else { + fatalError("Player item is not initialized.") + } + self.playerItem = currentItem + } + + private func setupAudioTap() { + // Create an AVMutableAudioMix + let audioMix = AVMutableAudioMix() + + // Get the first audio track + guard let audioTrack = playerItem.asset.tracks(withMediaType: .audio).first else { + print("No audio track found") + return + } + + // Create AVMutableAudioMixInputParameters for the track + let inputParams = AVMutableAudioMixInputParameters(track: audioTrack) + + // Install tap + inputParams.setVolume(1.0, at: .zero) + inputParams.audioTapProcessor = createAudioTapProcessor() + + audioMix.inputParameters = [inputParams] + + // Set the audio mix to the player item + playerItem.audioMix = audioMix + } + + private func removeAudioTap() { + playerItem.audioMix = nil + } + + private func createAudioTapProcessor() -> MTAudioProcessingTap { + var callbacks = MTAudioProcessingTapCallbacks( + version: kMTAudioProcessingTapCallbacksVersion_0, + clientInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()), + init: tapInitCallback, + finalize: tapFinalizeCallback, + prepare: tapPrepareCallback, + unprepare: tapUnprepareCallback, + process: tapProcessCallback + ) + + var tap: Unmanaged? + let status = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap) + + if status != noErr { + print("Error creating MTAudioProcessingTap: \(status)") + } + + return tap!.takeRetainedValue() + } + + // Define the TapContext class + class TapContext { + var processingFormat: AVAudioFormat? + weak var selfInstance: AudioEngineFromVideo? + } + + // MTAudioProcessingTap callbacks + private let tapInitCallback: MTAudioProcessingTapInitCallback = { (tap, clientInfo, tapStorageOut) in + // Initialization code + let context = TapContext() + context.selfInstance = Unmanaged.fromOpaque(clientInfo!).takeUnretainedValue() + tapStorageOut.pointee = Unmanaged.passRetained(context).toOpaque() + } + + private let tapFinalizeCallback: MTAudioProcessingTapFinalizeCallback = { (tap) in + // Finalization code + let storage = MTAudioProcessingTapGetStorage(tap) + Unmanaged.fromOpaque(storage).release() + } + + private let tapPrepareCallback: MTAudioProcessingTapPrepareCallback = { (tap, maxFrames, processingFormat) in + // Prepare code + let storage = MTAudioProcessingTapGetStorage(tap) + let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() + // Save the processing format + var asbd = processingFormat.pointee + context.processingFormat = AVAudioFormat(streamDescription: &asbd) + } + + private let tapUnprepareCallback: MTAudioProcessingTapUnprepareCallback = { (tap) in + // Unprepare code if needed + } + + private let tapProcessCallback: MTAudioProcessingTapProcessCallback = { (tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) in + let storage = MTAudioProcessingTapGetStorage(tap) + let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() + guard let selfInstance = context.selfInstance else { + print("Self instance not available") + return + } + + var status = noErr + var tapFlags: MTAudioProcessingTapFlags = 0 + var numFrames = numberFrames + var timeStamp = CMTimeRange() + + // Get source audio and timestamp + status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, &tapFlags, &timeStamp, &numFrames) + if status != noErr { + print("Error getting source audio: \(status)") + return + } + + // Convert AudioTimeStamp to AVAudioTime + guard let processingFormat = context.processingFormat else { + print("Processing format not available") + return + } + + let sampleRate = processingFormat.sampleRate + let audioTime = AVAudioTime(hostTime: mach_absolute_time()) + + // Process audio bufferListInOut + let bufferListPtr = UnsafeMutableAudioBufferListPointer(bufferListInOut) + // Now you can access audio samples from bufferListPtr + + // Create an AVAudioPCMBuffer and pass it to the recorder + if let pcmBuffer = AVAudioPCMBuffer(pcmFormat: processingFormat, frameCapacity: AVAudioFrameCount(numFrames)) { + pcmBuffer.frameLength = AVAudioFrameCount(numFrames) + for i in 0.. Date: Wed, 20 Nov 2024 12:59:35 +0100 Subject: [PATCH 16/26] Fix issue that prevented the audio to be heard while recording. --- Sources/AudioEngine/AudioEngineFromVideo.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Sources/AudioEngine/AudioEngineFromVideo.swift b/Sources/AudioEngine/AudioEngineFromVideo.swift index 43d2481..e1466fa 100644 --- a/Sources/AudioEngine/AudioEngineFromVideo.swift +++ b/Sources/AudioEngine/AudioEngineFromVideo.swift @@ -133,6 +133,10 @@ public final class AudioEngineFromVideo { return } + // **Set the number of frames and flags for output** + numberFramesOut.pointee = numFrames + flagsOut.pointee = tapFlags + // Convert AudioTimeStamp to AVAudioTime guard let processingFormat = context.processingFormat else { print("Processing format not available") @@ -164,6 +168,8 @@ public final class AudioEngineFromVideo { } } + + func startPlayback() { player.play() } From 3fb9188d32b8c39d174076d2282bbd3f85674fc9 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Wed, 20 Nov 2024 16:16:47 +0100 Subject: [PATCH 17/26] Refactoring the class. Change name of the class and rename CMTimeRange timeRangeOut variable inside the MTAudioProcessingTapProcessCallback. --- .../AudioEngine.AVPlayerTapper.swift | 176 +++++++++++++++++ .../AudioEngine/AudioEngineFromVideo.swift | 181 ------------------ 2 files changed, 176 insertions(+), 181 deletions(-) create mode 100644 Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift delete mode 100644 Sources/AudioEngine/AudioEngineFromVideo.swift diff --git a/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift b/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift new file mode 100644 index 0000000..fa8df48 --- /dev/null +++ b/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift @@ -0,0 +1,176 @@ +import Foundation +import AVFoundation + +extension AudioEngine { + public final class AVPlayerTapper { + private weak var player: AVPlayer? + private var playerItem: AVPlayerItem + private var audioFormat: AVAudioFormat? = AVAudioFormat(standardFormatWithSampleRate: 44100, channels: 2) + + @SCNObservable public internal(set) var error: Swift.Error? + + public weak var recorder: BaseRecorder? { + didSet { + oldValue?.audioInput.audioFormat = nil + guard let recorder = recorder else { + removeAudioTap() + return + } + + recorder.audioInput.audioFormat = audioFormat + + guard oldValue == nil else { return } + removeAudioTap() + setupAudioTap() + } + } + + deinit { + recorder = nil + } + + public init(player: AVPlayer) { + self.player = player + guard let currentItem = player.currentItem else { + fatalError("Player item is not initialized.") + } + self.playerItem = currentItem + } + + private func setupAudioTap() { + // Create an AVMutableAudioMix + let audioMix = AVMutableAudioMix() + + // Get the first audio track + guard let audioTrack = playerItem.asset.tracks(withMediaType: .audio).first else { + print("No audio track found") + return + } + + // Create AVMutableAudioMixInputParameters for the track + let inputParams = AVMutableAudioMixInputParameters(track: audioTrack) + + // Install tap + inputParams.setVolume(1.0, at: .zero) + inputParams.audioTapProcessor = createAudioTapProcessor() + + audioMix.inputParameters = [inputParams] + + // Set the audio mix to the player item + playerItem.audioMix = audioMix + } + + private func removeAudioTap() { + playerItem.audioMix = nil + } + + private func createAudioTapProcessor() -> MTAudioProcessingTap { + var callbacks = MTAudioProcessingTapCallbacks( + version: kMTAudioProcessingTapCallbacksVersion_0, + clientInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()), + init: tapInitCallback, + finalize: tapFinalizeCallback, + prepare: tapPrepareCallback, + unprepare: tapUnprepareCallback, + process: tapProcessCallback + ) + + var tap: Unmanaged? + let status = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap) + + if status != noErr { + print("Error creating MTAudioProcessingTap: \(status)") + } + + return tap!.takeRetainedValue() + } + + // Define the TapContext class + class TapContext { + var processingFormat: AVAudioFormat? + weak var selfInstance: AudioEngine.AVPlayerTapper? + } + + // MTAudioProcessingTap callbacks + private let tapInitCallback: MTAudioProcessingTapInitCallback = { (tap, clientInfo, tapStorageOut) in + // Initialization code + let context = TapContext() + context.selfInstance = Unmanaged.fromOpaque(clientInfo!).takeUnretainedValue() + tapStorageOut.pointee = Unmanaged.passRetained(context).toOpaque() + } + + private let tapFinalizeCallback: MTAudioProcessingTapFinalizeCallback = { (tap) in + // Finalization code + let storage = MTAudioProcessingTapGetStorage(tap) + Unmanaged.fromOpaque(storage).release() + } + + private let tapPrepareCallback: MTAudioProcessingTapPrepareCallback = { (tap, maxFrames, processingFormat) in + // Prepare code + let storage = MTAudioProcessingTapGetStorage(tap) + let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() + // Save the processing format + var asbd = processingFormat.pointee + context.processingFormat = AVAudioFormat(streamDescription: &asbd) + } + + private let tapUnprepareCallback: MTAudioProcessingTapUnprepareCallback = { (tap) in + // Unprepare code if needed + } + + private let tapProcessCallback: MTAudioProcessingTapProcessCallback = { (tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) in + let storage = MTAudioProcessingTapGetStorage(tap) + let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() + guard let selfInstance = context.selfInstance else { + print("Self instance not available") + return + } + + var status = noErr + var tapFlags: MTAudioProcessingTapFlags = 0 + var numFrames = numberFrames + var timeRangeOut = CMTimeRange() + + // Get source audio and timestamp + status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, &tapFlags, &timeRangeOut, &numFrames) + if status != noErr { + print("Error getting source audio: \(status)") + return + } + + // **Set the number of frames and flags for output** + numberFramesOut.pointee = numFrames + flagsOut.pointee = tapFlags + + // Convert AudioTimeStamp to AVAudioTime + guard let processingFormat = context.processingFormat else { + print("Processing format not available") + return + } + + let sampleRate = processingFormat.sampleRate + let audioTime = AVAudioTime(hostTime: mach_absolute_time()) + + // Process audio bufferListInOut + let bufferListPtr = UnsafeMutableAudioBufferListPointer(bufferListInOut) + // Now you can access audio samples from bufferListPtr + + // Create an AVAudioPCMBuffer and pass it to the recorder + if let pcmBuffer = AVAudioPCMBuffer(pcmFormat: processingFormat, frameCapacity: AVAudioFrameCount(numFrames)) { + pcmBuffer.frameLength = AVAudioFrameCount(numFrames) + for i in 0.. MTAudioProcessingTap { - var callbacks = MTAudioProcessingTapCallbacks( - version: kMTAudioProcessingTapCallbacksVersion_0, - clientInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()), - init: tapInitCallback, - finalize: tapFinalizeCallback, - prepare: tapPrepareCallback, - unprepare: tapUnprepareCallback, - process: tapProcessCallback - ) - - var tap: Unmanaged? - let status = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap) - - if status != noErr { - print("Error creating MTAudioProcessingTap: \(status)") - } - - return tap!.takeRetainedValue() - } - - // Define the TapContext class - class TapContext { - var processingFormat: AVAudioFormat? - weak var selfInstance: AudioEngineFromVideo? - } - - // MTAudioProcessingTap callbacks - private let tapInitCallback: MTAudioProcessingTapInitCallback = { (tap, clientInfo, tapStorageOut) in - // Initialization code - let context = TapContext() - context.selfInstance = Unmanaged.fromOpaque(clientInfo!).takeUnretainedValue() - tapStorageOut.pointee = Unmanaged.passRetained(context).toOpaque() - } - - private let tapFinalizeCallback: MTAudioProcessingTapFinalizeCallback = { (tap) in - // Finalization code - let storage = MTAudioProcessingTapGetStorage(tap) - Unmanaged.fromOpaque(storage).release() - } - - private let tapPrepareCallback: MTAudioProcessingTapPrepareCallback = { (tap, maxFrames, processingFormat) in - // Prepare code - let storage = MTAudioProcessingTapGetStorage(tap) - let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() - // Save the processing format - var asbd = processingFormat.pointee - context.processingFormat = AVAudioFormat(streamDescription: &asbd) - } - - private let tapUnprepareCallback: MTAudioProcessingTapUnprepareCallback = { (tap) in - // Unprepare code if needed - } - - private let tapProcessCallback: MTAudioProcessingTapProcessCallback = { (tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) in - let storage = MTAudioProcessingTapGetStorage(tap) - let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() - guard let selfInstance = context.selfInstance else { - print("Self instance not available") - return - } - - var status = noErr - var tapFlags: MTAudioProcessingTapFlags = 0 - var numFrames = numberFrames - var timeStamp = CMTimeRange() - - // Get source audio and timestamp - status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, &tapFlags, &timeStamp, &numFrames) - if status != noErr { - print("Error getting source audio: \(status)") - return - } - - // **Set the number of frames and flags for output** - numberFramesOut.pointee = numFrames - flagsOut.pointee = tapFlags - - // Convert AudioTimeStamp to AVAudioTime - guard let processingFormat = context.processingFormat else { - print("Processing format not available") - return - } - - let sampleRate = processingFormat.sampleRate - let audioTime = AVAudioTime(hostTime: mach_absolute_time()) - - // Process audio bufferListInOut - let bufferListPtr = UnsafeMutableAudioBufferListPointer(bufferListInOut) - // Now you can access audio samples from bufferListPtr - - // Create an AVAudioPCMBuffer and pass it to the recorder - if let pcmBuffer = AVAudioPCMBuffer(pcmFormat: processingFormat, frameCapacity: AVAudioFrameCount(numFrames)) { - pcmBuffer.frameLength = AVAudioFrameCount(numFrames) - for i in 0.. Date: Wed, 20 Nov 2024 16:32:30 +0100 Subject: [PATCH 18/26] Remove the reference to the playerItem. --- .../AudioEngine.AVPlayerTapper.swift | 24 +++++++++---------- .../BaseRecorder.AudioInput.swift | 8 ++----- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift b/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift index fa8df48..95f6373 100644 --- a/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift +++ b/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift @@ -4,7 +4,6 @@ import AVFoundation extension AudioEngine { public final class AVPlayerTapper { private weak var player: AVPlayer? - private var playerItem: AVPlayerItem private var audioFormat: AVAudioFormat? = AVAudioFormat(standardFormatWithSampleRate: 44100, channels: 2) @SCNObservable public internal(set) var error: Swift.Error? @@ -31,10 +30,9 @@ extension AudioEngine { public init(player: AVPlayer) { self.player = player - guard let currentItem = player.currentItem else { - fatalError("Player item is not initialized.") + guard player.currentItem != nil else { + fatalError("FATAL: Player item is not initialized.") } - self.playerItem = currentItem } private func setupAudioTap() { @@ -42,8 +40,8 @@ extension AudioEngine { let audioMix = AVMutableAudioMix() // Get the first audio track - guard let audioTrack = playerItem.asset.tracks(withMediaType: .audio).first else { - print("No audio track found") + guard let audioTrack = player?.currentItem?.asset.tracks(withMediaType: .audio).first else { + print("ERROR: No audio track found") return } @@ -57,11 +55,11 @@ extension AudioEngine { audioMix.inputParameters = [inputParams] // Set the audio mix to the player item - playerItem.audioMix = audioMix + player?.currentItem?.audioMix = audioMix } private func removeAudioTap() { - playerItem.audioMix = nil + player?.currentItem?.audioMix = nil } private func createAudioTapProcessor() -> MTAudioProcessingTap { @@ -79,7 +77,7 @@ extension AudioEngine { let status = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap) if status != noErr { - print("Error creating MTAudioProcessingTap: \(status)") + fatalError("FATAL: creating MTAudioProcessingTap: \(status)") } return tap!.takeRetainedValue() @@ -122,7 +120,7 @@ extension AudioEngine { let storage = MTAudioProcessingTapGetStorage(tap) let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() guard let selfInstance = context.selfInstance else { - print("Self instance not available") + print("ERROR: tapProcessCallback: selfInstance not available") return } @@ -134,7 +132,7 @@ extension AudioEngine { // Get source audio and timestamp status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, &tapFlags, &timeRangeOut, &numFrames) if status != noErr { - print("Error getting source audio: \(status)") + print("ERROR: tapProcessCallback: getting source audio, MTAudioProcessingTapGetSourceAudio status: \(status)") return } @@ -144,7 +142,7 @@ extension AudioEngine { // Convert AudioTimeStamp to AVAudioTime guard let processingFormat = context.processingFormat else { - print("Processing format not available") + print("ERROR: tapProcessCallback: Processing format is not available") return } @@ -167,7 +165,7 @@ extension AudioEngine { let sampleBuffer = try AudioEngine.createAudioSampleBuffer(from: pcmBuffer, time: audioTime) selfInstance.recorder?.audioInput.audioEngine(didOutputAudioSampleBuffer: sampleBuffer) } catch let error { - print("ERROR: error creating audio sample buffer: \(error)") + print("ERROR: tapProcessCallback: error creating audio sample buffer: \(error)") selfInstance.error = error } } diff --git a/Sources/Recorder/BaseRecorder/BaseRecorder.AudioInput.swift b/Sources/Recorder/BaseRecorder/BaseRecorder.AudioInput.swift index 0be979d..ffa69f1 100644 --- a/Sources/Recorder/BaseRecorder/BaseRecorder.AudioInput.swift +++ b/Sources/Recorder/BaseRecorder/BaseRecorder.AudioInput.swift @@ -88,14 +88,10 @@ extension BaseRecorder.AudioInput: ARSessionObserver { @available(iOS 13.0, *) extension BaseRecorder.AudioInput { - + func audioEngine(didOutputAudioSampleBuffer audioSampleBuffer: CMSampleBuffer ) { - guard started, useAudioEngine else { - print("INFO: Ignoring audioSampleBuffer because started is \(started)") - return - } - print("INFO: audioEngine didOutputAudioSampleBuffer") + guard started, useAudioEngine else { return } queue.async { [output] in output?(audioSampleBuffer) } } } From 28eba163bc7a96285e9baeaeee91a9ca505b9b2a Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Tue, 25 Mar 2025 22:43:22 +0100 Subject: [PATCH 19/26] Pause video before finish recording to avoid black frames at the end of the video. --- SCNRecorder.podspec | 1 + Sources/Outputs/VideoOutput/VideoOutput.swift | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/SCNRecorder.podspec b/SCNRecorder.podspec index 0bc21ed..1d9904e 100644 --- a/SCNRecorder.podspec +++ b/SCNRecorder.podspec @@ -10,6 +10,7 @@ Pod::Spec.new do |s| s.module_name = 'SCNRecorder' s.swift_version = '5.0' s.source_files = 'Sources/**/*.{swift}' + s.exclude_files = 'Sources/Extensions/AVCaptureSession+BaseRecorder.swift' s.dependency 'MTDMulticastDelegate' s.app_spec 'Example' do |app_spec| diff --git a/Sources/Outputs/VideoOutput/VideoOutput.swift b/Sources/Outputs/VideoOutput/VideoOutput.swift index 23d57f6..e8f748b 100644 --- a/Sources/Outputs/VideoOutput/VideoOutput.swift +++ b/Sources/Outputs/VideoOutput/VideoOutput.swift @@ -206,7 +206,10 @@ extension VideoOutput { } func finish(completionHandler handler: @escaping () -> Void) { - queue.async { [weak self] in self?.unsafeFinish(completionHandler: handler) } + queue.async { [weak self] in + self?.unsafePause() + self?.unsafeFinish(completionHandler: handler) + } } func cancel() { From ac9083386e97075a6fba9deff64069ed4d128fb1 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Tue, 10 Jun 2025 15:32:27 +0200 Subject: [PATCH 20/26] Just add some comments explaining what the swizzled_nextDrawable method does. --- Sources/RecordableLayer/CAMetalLayer+RecordableLayer.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/RecordableLayer/CAMetalLayer+RecordableLayer.swift b/Sources/RecordableLayer/CAMetalLayer+RecordableLayer.swift index ccecbe2..0f944b7 100644 --- a/Sources/RecordableLayer/CAMetalLayer+RecordableLayer.swift +++ b/Sources/RecordableLayer/CAMetalLayer+RecordableLayer.swift @@ -60,9 +60,9 @@ extension CAMetalLayer: RecordableLayer { } @objc dynamic func swizzled_nextDrawable() -> CAMetalDrawable? { - let nextDrawable = swizzled_nextDrawable() - lastTexture = nextDrawable?.texture - return nextDrawable + let nextDrawable = swizzled_nextDrawable() //It calls swizzled_nextDrawable(), which due to the swizzle now actually invokes the original nextDrawable method. + lastTexture = nextDrawable?.texture //It extracts the texture from the returned drawable and stores it in the lastTexture property. + return nextDrawable //Finally, it returns the drawable as the original method would } } From b101540985f9894e03a7893a56b5b9980dedd163 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Wed, 2 Jul 2025 23:15:11 +0200 Subject: [PATCH 21/26] Adjustments to align to master branch v2.9.0 --- .../VideoOutput/VideoOutput.State.swift | 55 ++++++++++++------- Sources/Outputs/VideoOutput/VideoOutput.swift | 20 +++++-- .../CAMetalLayer+RecordableLayer.swift | 6 +- Sources/SelfRecordable/SelfRecordable.swift | 8 +++ 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/Sources/Outputs/VideoOutput/VideoOutput.State.swift b/Sources/Outputs/VideoOutput/VideoOutput.State.swift index 4a65fd5..a6a6942 100644 --- a/Sources/Outputs/VideoOutput/VideoOutput.State.swift +++ b/Sources/Outputs/VideoOutput/VideoOutput.State.swift @@ -61,12 +61,10 @@ public enum VideoOutputState: Equatable { /// Recording is active. /// Appending audio and video frames. - case recording(time: CMTime) + case recording(time: CMTime, pause: CMTime?, resume: CMTime?) /// Recording is paused. - /// Not tested. - /// According to Apple, documentation might not work. - case paused + case paused(time: CMTime, pause: CMTime) /// Recording is canceled. /// Final state, not further actions are possible. @@ -107,11 +105,12 @@ public enum VideoOutputState: Equatable { .preparing: return .preparing - case .recording(let time): - return .recording(time: time) + case let .recording(time, pause, resume): + return .recording(time: time, pause: pause, resume: resume) - case .paused: - return .preparing + case let .paused(time, pause): + let resume = CMTime(seconds: CACurrentMediaTime(), preferredTimescale: pause.timescale) + return .recording(time: time, pause: pause, resume: resume) case .canceled, .finished, @@ -130,9 +129,14 @@ public enum VideoOutputState: Equatable { .preparing: return .ready - case .recording(let seconds): - videoOutput.endSession(at: seconds) - return .paused + case let .recording(lastTime, pause, resume): + let current = CMTime(seconds: CACurrentMediaTime(), preferredTimescale: lastTime.timescale) + if let pause, let resume { + let adjustedPause = adjustTime(current: current, resume: resume, pause: pause) + return .paused(time: lastTime, pause: adjustedPause) + } else { + return .paused(time: lastTime, pause: current) + } case .paused, .canceled, @@ -229,14 +233,13 @@ private extension VideoOutputState { } else { time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) } - videoOutput.startSession(at: time) try videoOutput.appendVideo(sampleBuffer: sampleBuffer) - return .recording(time: time) + return .recording(time: time, pause: nil, resume: nil) - case .recording(let time): + case let .recording(time, pause, resume): try videoOutput.appendVideo(sampleBuffer: sampleBuffer) - return .recording(time: time) + return .recording(time: time, pause: pause, resume: resume) case .paused, .canceled, @@ -260,11 +263,17 @@ private extension VideoOutputState { case .preparing: videoOutput.startSession(at: time) try videoOutput.append(pixelBuffer: pixelBuffer, withPresentationTime: time) - return .recording(time: time) + return .recording(time: time, pause: nil, resume: nil) - case .recording: - try videoOutput.append(pixelBuffer: pixelBuffer, withPresentationTime: time) - return .recording(time: time) + case let .recording(_ ,pause, resume): + let finalTime: CMTime + if let pause, let resume { + finalTime = adjustTime(current: time, resume: resume, pause: pause) + } else { + finalTime = time + } + try videoOutput.append(pixelBuffer: pixelBuffer, withPresentationTime: finalTime) + return .recording(time: finalTime, pause: pause, resume: resume) case .paused, .canceled, @@ -285,9 +294,9 @@ private extension VideoOutputState { .preparing: return self - case .recording(let time): + case let .recording(time, pause, resume): try videoOutput.appendAudio(sampleBuffer: sampleBuffer) - return .recording(time: time) + return .recording(time: time, pause: pause, resume: resume) case .paused, .canceled, @@ -296,4 +305,8 @@ private extension VideoOutputState { return self } } + + func adjustTime(current: CMTime, resume: CMTime, pause: CMTime) -> CMTime { + return CMTimeSubtract(current, CMTimeSubtract(resume, pause)) + } } diff --git a/Sources/Outputs/VideoOutput/VideoOutput.swift b/Sources/Outputs/VideoOutput/VideoOutput.swift index e8f748b..ef04378 100644 --- a/Sources/Outputs/VideoOutput/VideoOutput.swift +++ b/Sources/Outputs/VideoOutput/VideoOutput.swift @@ -143,7 +143,9 @@ extension VideoOutput { } func append(pixelBuffer: CVPixelBuffer, withPresentationTime time: CMTime) throws { - guard pixelBufferAdaptor.assetWriterInput.isReadyForMoreMediaData else { return } + guard pixelBufferAdaptor.assetWriterInput.isReadyForMoreMediaData else { + return + } guard pixelBufferAdaptor.append(pixelBuffer, withPresentationTime: time) else { if assetWriter.status == .failed { throw assetWriter.error ?? Error.unknown } return @@ -155,7 +157,9 @@ extension VideoOutput { } func appendVideo(sampleBuffer: CMSampleBuffer) throws { - guard videoInput.isReadyForMoreMediaData else { return } + guard videoInput.isReadyForMoreMediaData else { + return + } guard videoInput.append(sampleBuffer) else { if assetWriter.status == .failed { throw assetWriter.error ?? Error.unknown } return @@ -177,10 +181,16 @@ extension VideoOutput { } func appendAudio(sampleBuffer: CMSampleBuffer) throws { - guard let audioInput = audioInput else { return } - guard audioInput.isReadyForMoreMediaData else { return } + guard + let audioInput = audioInput, + audioInput.isReadyForMoreMediaData + else { + return + } guard audioInput.append(sampleBuffer) else { - if assetWriter.status == .failed { throw assetWriter.error ?? Error.unknown } + if assetWriter.status == .failed { + throw assetWriter.error ?? Error.unknown + } return } } diff --git a/Sources/RecordableLayer/CAMetalLayer+RecordableLayer.swift b/Sources/RecordableLayer/CAMetalLayer+RecordableLayer.swift index 0f944b7..ccecbe2 100644 --- a/Sources/RecordableLayer/CAMetalLayer+RecordableLayer.swift +++ b/Sources/RecordableLayer/CAMetalLayer+RecordableLayer.swift @@ -60,9 +60,9 @@ extension CAMetalLayer: RecordableLayer { } @objc dynamic func swizzled_nextDrawable() -> CAMetalDrawable? { - let nextDrawable = swizzled_nextDrawable() //It calls swizzled_nextDrawable(), which due to the swizzle now actually invokes the original nextDrawable method. - lastTexture = nextDrawable?.texture //It extracts the texture from the returned drawable and stores it in the lastTexture property. - return nextDrawable //Finally, it returns the drawable as the original method would + let nextDrawable = swizzled_nextDrawable() + lastTexture = nextDrawable?.texture + return nextDrawable } } diff --git a/Sources/SelfRecordable/SelfRecordable.swift b/Sources/SelfRecordable/SelfRecordable.swift index 5838815..f55e270 100644 --- a/Sources/SelfRecordable/SelfRecordable.swift +++ b/Sources/SelfRecordable/SelfRecordable.swift @@ -189,6 +189,14 @@ public extension SelfRecordable { return videoRecording } + func pauseVideoRecording() { + videoRecording?.pause() + } + + func resumeVideoRecording() { + videoRecording?.resume() + } + func finishVideoRecording(completionHandler handler: @escaping (VideoRecording.Info) -> Void) { videoRecording?.finish { videoRecordingInfo in DispatchQueue.main.async { handler(videoRecordingInfo) } From ba9edbe02e95b99e3cfe27b12542cfb7ddc5a5d7 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Thu, 3 Jul 2025 16:40:05 +0200 Subject: [PATCH 22/26] Remove the unsafe pause that was introduced to solve the black frame issue when the endSession was done on pause. --- Sources/Outputs/VideoOutput/VideoOutput.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Sources/Outputs/VideoOutput/VideoOutput.swift b/Sources/Outputs/VideoOutput/VideoOutput.swift index ef04378..fc4a276 100644 --- a/Sources/Outputs/VideoOutput/VideoOutput.swift +++ b/Sources/Outputs/VideoOutput/VideoOutput.swift @@ -217,7 +217,6 @@ extension VideoOutput { func finish(completionHandler handler: @escaping () -> Void) { queue.async { [weak self] in - self?.unsafePause() self?.unsafeFinish(completionHandler: handler) } } From 7e8009e82caf9e813e62296b86ac9549308d4f45 Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Thu, 3 Jul 2025 16:40:51 +0200 Subject: [PATCH 23/26] End the session before finishWriting to avoid black frames at the end of recording. --- Sources/Outputs/VideoOutput/VideoOutput.State.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/Outputs/VideoOutput/VideoOutput.State.swift b/Sources/Outputs/VideoOutput/VideoOutput.State.swift index a6a6942..838a3a3 100644 --- a/Sources/Outputs/VideoOutput/VideoOutput.State.swift +++ b/Sources/Outputs/VideoOutput/VideoOutput.State.swift @@ -155,8 +155,8 @@ public enum VideoOutputState: Equatable { handler() return .canceled - case .recording, - .paused: + case let .recording (currentTime, _, _), let .paused(currentTime, _): + videoOutput.endSession(at: currentTime) videoOutput.finishWriting(completionHandler: handler) return .finished From cd7257b7840bc75451a27880dceda381cfbb845b Mon Sep 17 00:00:00 2001 From: Giovanni Murru Date: Wed, 1 Oct 2025 16:19:52 +0200 Subject: [PATCH 24/26] Remove use of Unmanaged in MTAudioProcessingTap to fix MTAudioProcessingTapCreate parameter type error. --- Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift b/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift index 95f6373..4d91f41 100644 --- a/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift +++ b/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift @@ -73,14 +73,14 @@ extension AudioEngine { process: tapProcessCallback ) - var tap: Unmanaged? + var tap: MTAudioProcessingTap? let status = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap) if status != noErr { fatalError("FATAL: creating MTAudioProcessingTap: \(status)") } - return tap!.takeRetainedValue() + return tap! } // Define the TapContext class From 21e3c4b5df24d85c53c87e2a9725426a7b3aa7ce Mon Sep 17 00:00:00 2001 From: Hassan Taleb Date: Thu, 29 Jan 2026 14:47:39 +0200 Subject: [PATCH 25/26] Update minimum iOS version to 13 --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index a82452c..d1d9733 100644 --- a/Package.swift +++ b/Package.swift @@ -4,7 +4,7 @@ import PackageDescription let package = Package( name: "SCNRecorder", - platforms: [ .iOS(.v12) ], + platforms: [ .iOS(.v13) ], products: [ .library( name: "SCNRecorder", From c755e5a4b032ae448b4d4a25e533153eb774d6fe Mon Sep 17 00:00:00 2001 From: Hassan Taleb Date: Mon, 27 Apr 2026 12:44:42 +0300 Subject: [PATCH 26/26] Fix AVPlayer audio tap crash and improve AudioEngine processing format handling --- .../AudioEngine.AVPlayerTapper.swift | 382 +++++++------ Sources/AudioEngine/AudioEngine.swift | 524 +++++++++--------- 2 files changed, 482 insertions(+), 424 deletions(-) diff --git a/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift b/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift index 4d91f41..0f11086 100644 --- a/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift +++ b/Sources/AudioEngine/AudioEngine.AVPlayerTapper.swift @@ -2,173 +2,231 @@ import Foundation import AVFoundation extension AudioEngine { - public final class AVPlayerTapper { - private weak var player: AVPlayer? - private var audioFormat: AVAudioFormat? = AVAudioFormat(standardFormatWithSampleRate: 44100, channels: 2) - - @SCNObservable public internal(set) var error: Swift.Error? - - public weak var recorder: BaseRecorder? { - didSet { - oldValue?.audioInput.audioFormat = nil - guard let recorder = recorder else { - removeAudioTap() - return + public final class AVPlayerTapper { + private weak var player: AVPlayer? + private var audioFormat: AVAudioFormat? { + AVAudioFormat(commonFormat: .pcmFormatFloat32, + sampleRate: 44100, + channels: 2, + interleaved: false) } - recorder.audioInput.audioFormat = audioFormat + @SCNObservable public internal(set) var error: Swift.Error? - guard oldValue == nil else { return } - removeAudioTap() - setupAudioTap() - } - } - - deinit { - recorder = nil - } - - public init(player: AVPlayer) { - self.player = player - guard player.currentItem != nil else { - fatalError("FATAL: Player item is not initialized.") - } - } - - private func setupAudioTap() { - // Create an AVMutableAudioMix - let audioMix = AVMutableAudioMix() - - // Get the first audio track - guard let audioTrack = player?.currentItem?.asset.tracks(withMediaType: .audio).first else { - print("ERROR: No audio track found") - return - } - - // Create AVMutableAudioMixInputParameters for the track - let inputParams = AVMutableAudioMixInputParameters(track: audioTrack) - - // Install tap - inputParams.setVolume(1.0, at: .zero) - inputParams.audioTapProcessor = createAudioTapProcessor() - - audioMix.inputParameters = [inputParams] - - // Set the audio mix to the player item - player?.currentItem?.audioMix = audioMix - } - - private func removeAudioTap() { - player?.currentItem?.audioMix = nil - } - - private func createAudioTapProcessor() -> MTAudioProcessingTap { - var callbacks = MTAudioProcessingTapCallbacks( - version: kMTAudioProcessingTapCallbacksVersion_0, - clientInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()), - init: tapInitCallback, - finalize: tapFinalizeCallback, - prepare: tapPrepareCallback, - unprepare: tapUnprepareCallback, - process: tapProcessCallback - ) - - var tap: MTAudioProcessingTap? - let status = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap) - - if status != noErr { - fatalError("FATAL: creating MTAudioProcessingTap: \(status)") - } - - return tap! - } - - // Define the TapContext class - class TapContext { - var processingFormat: AVAudioFormat? - weak var selfInstance: AudioEngine.AVPlayerTapper? - } - - // MTAudioProcessingTap callbacks - private let tapInitCallback: MTAudioProcessingTapInitCallback = { (tap, clientInfo, tapStorageOut) in - // Initialization code - let context = TapContext() - context.selfInstance = Unmanaged.fromOpaque(clientInfo!).takeUnretainedValue() - tapStorageOut.pointee = Unmanaged.passRetained(context).toOpaque() - } - - private let tapFinalizeCallback: MTAudioProcessingTapFinalizeCallback = { (tap) in - // Finalization code - let storage = MTAudioProcessingTapGetStorage(tap) - Unmanaged.fromOpaque(storage).release() - } - - private let tapPrepareCallback: MTAudioProcessingTapPrepareCallback = { (tap, maxFrames, processingFormat) in - // Prepare code - let storage = MTAudioProcessingTapGetStorage(tap) - let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() - // Save the processing format - var asbd = processingFormat.pointee - context.processingFormat = AVAudioFormat(streamDescription: &asbd) - } - - private let tapUnprepareCallback: MTAudioProcessingTapUnprepareCallback = { (tap) in - // Unprepare code if needed - } - - private let tapProcessCallback: MTAudioProcessingTapProcessCallback = { (tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) in - let storage = MTAudioProcessingTapGetStorage(tap) - let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() - guard let selfInstance = context.selfInstance else { - print("ERROR: tapProcessCallback: selfInstance not available") - return - } - - var status = noErr - var tapFlags: MTAudioProcessingTapFlags = 0 - var numFrames = numberFrames - var timeRangeOut = CMTimeRange() - - // Get source audio and timestamp - status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, &tapFlags, &timeRangeOut, &numFrames) - if status != noErr { - print("ERROR: tapProcessCallback: getting source audio, MTAudioProcessingTapGetSourceAudio status: \(status)") - return - } - - // **Set the number of frames and flags for output** - numberFramesOut.pointee = numFrames - flagsOut.pointee = tapFlags - - // Convert AudioTimeStamp to AVAudioTime - guard let processingFormat = context.processingFormat else { - print("ERROR: tapProcessCallback: Processing format is not available") - return - } - - let sampleRate = processingFormat.sampleRate - let audioTime = AVAudioTime(hostTime: mach_absolute_time()) - - // Process audio bufferListInOut - let bufferListPtr = UnsafeMutableAudioBufferListPointer(bufferListInOut) - // Now you can access audio samples from bufferListPtr - - // Create an AVAudioPCMBuffer and pass it to the recorder - if let pcmBuffer = AVAudioPCMBuffer(pcmFormat: processingFormat, frameCapacity: AVAudioFrameCount(numFrames)) { - pcmBuffer.frameLength = AVAudioFrameCount(numFrames) - for i in 0.. MTAudioProcessingTap { + var callbacks = MTAudioProcessingTapCallbacks( + version: kMTAudioProcessingTapCallbacksVersion_0, + clientInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()), + init: tapInitCallback, + finalize: tapFinalizeCallback, + prepare: tapPrepareCallback, + unprepare: tapUnprepareCallback, + process: tapProcessCallback + ) + + var tap: MTAudioProcessingTap? + let status = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap) + + if status != noErr { + fatalError("FATAL: creating MTAudioProcessingTap: \(status)") + } + + return tap! + } + + // Define the TapContext class + class TapContext { + var processingFormat: AVAudioFormat? + weak var selfInstance: AudioEngine.AVPlayerTapper? + } + + // MTAudioProcessingTap callbacks + private let tapInitCallback: MTAudioProcessingTapInitCallback = { (tap, clientInfo, tapStorageOut) in + // Initialization code + let context = TapContext() + context.selfInstance = Unmanaged.fromOpaque(clientInfo!).takeUnretainedValue() + tapStorageOut.pointee = Unmanaged.passRetained(context).toOpaque() + } + + private let tapFinalizeCallback: MTAudioProcessingTapFinalizeCallback = { (tap) in + // Finalization code + let storage = MTAudioProcessingTapGetStorage(tap) + Unmanaged.fromOpaque(storage).release() + } + + private let tapPrepareCallback: MTAudioProcessingTapPrepareCallback = { (tap, maxFrames, processingFormat) in + // Prepare code + let storage = MTAudioProcessingTapGetStorage(tap) + let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() + // Save the processing format + var asbd = processingFormat.pointee + context.processingFormat = AVAudioFormat(streamDescription: &asbd) + } + + private let tapUnprepareCallback: MTAudioProcessingTapUnprepareCallback = { (tap) in + // Unprepare code if needed } - do { - let sampleBuffer = try AudioEngine.createAudioSampleBuffer(from: pcmBuffer, time: audioTime) - selfInstance.recorder?.audioInput.audioEngine(didOutputAudioSampleBuffer: sampleBuffer) - } catch let error { - print("ERROR: tapProcessCallback: error creating audio sample buffer: \(error)") - selfInstance.error = error + private let tapProcessCallback: MTAudioProcessingTapProcessCallback = { + (tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) in + + let storage = MTAudioProcessingTapGetStorage(tap) + let context = Unmanaged.fromOpaque(storage).takeUnretainedValue() + + guard let selfInstance = context.selfInstance else { + print("ERROR: tapProcessCallback: selfInstance not available") + return + } + + var status = noErr + var tapFlags: MTAudioProcessingTapFlags = 0 + var numFrames = numberFrames + var timeRangeOut = CMTimeRange() + + // Get source audio + status = MTAudioProcessingTapGetSourceAudio( + tap, + numberFrames, + bufferListInOut, + &tapFlags, + &timeRangeOut, + &numFrames + ) + + if status != noErr { + print("ERROR: tapProcessCallback: getting source audio, status: \(status)") + return + } + + numberFramesOut.pointee = numFrames + flagsOut.pointee = tapFlags + + let bufferListPtr = UnsafeMutableAudioBufferListPointer(bufferListInOut) + + // MARK: - SAFE FORMAT HANDLING + let processingFormat: AVAudioFormat + + if let format = context.processingFormat { + processingFormat = format + } else { + let channels = min(2, bufferListPtr.count) + + guard let fallback = AVAudioFormat( + commonFormat: .pcmFormatFloat32, + sampleRate: 44100, + channels: AVAudioChannelCount(channels), + interleaved: false + ) else { + print("ERROR: cannot create fallback format") + return + } + + context.processingFormat = fallback + processingFormat = fallback + } + + let audioTime = AVAudioTime(hostTime: mach_absolute_time()) + + guard let pcmBuffer = AVAudioPCMBuffer( + pcmFormat: processingFormat, + frameCapacity: AVAudioFrameCount(numFrames) + ) else { + return + } + + pcmBuffer.frameLength = AVAudioFrameCount(numFrames) + + // MARK: - SAFE CHANNEL COPY (FIXED CRASH HERE) + let dstChannels = Int(processingFormat.channelCount) + let srcChannels = bufferListPtr.count + let channelsToCopy = min(dstChannels, srcChannels) + + for i in 0...size + ) + + memcpy(dst, src, byteSize) + } + + // MARK: - SEND TO RECORDER + do { + let sampleBuffer = try AudioEngine.createAudioSampleBuffer( + from: pcmBuffer, + time: audioTime + ) + + selfInstance.recorder?.audioInput.audioEngine( + didOutputAudioSampleBuffer: sampleBuffer + ) + + } catch { + print("ERROR: tapProcessCallback: failed to create sample buffer: \(error)") + selfInstance.error = error + } } - } } - } } diff --git a/Sources/AudioEngine/AudioEngine.swift b/Sources/AudioEngine/AudioEngine.swift index eb28d8e..b0e089d 100644 --- a/Sources/AudioEngine/AudioEngine.swift +++ b/Sources/AudioEngine/AudioEngine.swift @@ -29,283 +29,283 @@ import UIKit @available(iOS 13.0, *) public final class AudioEngine { - - enum State { - - case normal - - case interrupted(playing: Bool) - - var isInterrupted: Bool { - if case .interrupted = self { return true } - return false - } - - var shouldResume: Bool { - if case .interrupted(true) = self { return true } - return false - } - } - - let queue = DispatchQueue(label: "AudioEngine.Processing", qos: .userInteractive) - - lazy var engine: AVAudioEngine = { - let engine = AVAudioEngine() - engine.attach(playerNode) - engine.attach(raterNode) - return engine - }() - - lazy var playerNode = AVAudioPlayerNode() - - lazy var raterNode = AVAudioUnitTimePitch() - - let audioSession = AVAudioSession.sharedInstance() - - let notificationCenter = NotificationCenter.default - - var observers = [NSObjectProtocol]() - - var state: State = .normal - - public var player: Player? { - didSet { - guard player !== oldValue else { return } - if let oldPlayer = oldValue { oldPlayer.stop() } - - recorder?.audioInput.audioFormat = nil - guard let player = player else { return } - recorder?.audioInput.audioFormat = player.audioFormat - - engine.connect(playerNode, to: raterNode, format: player.audioFormat) - engine.connect(raterNode, to: engine.mainMixerNode, format: player.audioFormat) - - player.attach( - playerNode: playerNode, - raterNode: raterNode, - queue: queue - ) - - player.willStart = { [weak self] in - guard let self = self else { return false } - - let isInterrupted = DispatchQueue.main.sync { () -> Bool in - if self.state.isInterrupted { self.state = .interrupted(playing: true) } - return self.state.isInterrupted - } - guard !isInterrupted else { return false } - - do { - try self.activateAudioSession() - try self.engine.start() - return true + + enum State { + + case normal + + case interrupted(playing: Bool) + + var isInterrupted: Bool { + if case .interrupted = self { return true } + return false } - catch { - self.error = error - return false + + var shouldResume: Bool { + if case .interrupted(true) = self { return true } + return false } - } - - player.didPause = { [weak self] in - guard let self = self else { return } - self.engine.pause() - self.deactivateAudioSession() - } - - player.didStop = { [weak self] in - guard let self = self else { return } - - self.engine.stop() - self.engine.reset() - self.deactivateAudioSession() - } } - } - - public weak var recorder: BaseRecorder? { - didSet { - oldValue?.audioInput.audioFormat = nil - guard let recorder = recorder else { - engine.mainMixerNode.removeTap(onBus: 0) - return - } - - recorder.audioInput.audioFormat = player?.audioFormat - - guard oldValue == nil else { return } - engine.mainMixerNode.removeTap(onBus: 0) - engine.mainMixerNode.installTap( - onBus: 0, - bufferSize: 4096, - format: nil - ) { [weak self] (buffer, time) in - guard let self = self else { return } - guard let recorder = self.recorder else { - self.engine.mainMixerNode.removeTap(onBus: 0) - return - } - - do { - let sampleBuffer = try Self.createAudioSampleBuffer(from: buffer, time: time) - recorder.audioInput.audioEngine(didOutputAudioSampleBuffer: sampleBuffer) + + let queue = DispatchQueue(label: "AudioEngine.Processing", qos: .userInteractive) + + lazy var engine: AVAudioEngine = { + let engine = AVAudioEngine() + engine.attach(playerNode) + engine.attach(raterNode) + return engine + }() + + lazy var playerNode = AVAudioPlayerNode() + + lazy var raterNode = AVAudioUnitTimePitch() + + let audioSession = AVAudioSession.sharedInstance() + + let notificationCenter = NotificationCenter.default + + var observers = [NSObjectProtocol]() + + var state: State = .normal + + public var player: Player? { + didSet { + guard player !== oldValue else { return } + if let oldPlayer = oldValue { oldPlayer.stop() } + + recorder?.audioInput.audioFormat = nil + guard let player = player else { return } + recorder?.audioInput.audioFormat = player.audioFormat + + engine.connect(playerNode, to: raterNode, format: player.audioFormat) + engine.connect(raterNode, to: engine.mainMixerNode, format: player.audioFormat) + + player.attach( + playerNode: playerNode, + raterNode: raterNode, + queue: queue + ) + + player.willStart = { [weak self] in + guard let self = self else { return false } + + let isInterrupted = DispatchQueue.main.sync { () -> Bool in + if self.state.isInterrupted { self.state = .interrupted(playing: true) } + return self.state.isInterrupted + } + guard !isInterrupted else { return false } + + do { + try self.activateAudioSession() + try self.engine.start() + return true + } + catch { + self.error = error + return false + } + } + + player.didPause = { [weak self] in + guard let self = self else { return } + self.engine.pause() + self.deactivateAudioSession() + } + + player.didStop = { [weak self] in + guard let self = self else { return } + + self.engine.stop() + self.engine.reset() + self.deactivateAudioSession() + } } - catch { - self.error = error + } + + public weak var recorder: BaseRecorder? { + didSet { + oldValue?.audioInput.audioFormat = nil + guard let recorder = recorder else { + engine.mainMixerNode.removeTap(onBus: 0) + return + } + + recorder.audioInput.audioFormat = player?.audioFormat + + guard oldValue == nil else { return } + engine.mainMixerNode.removeTap(onBus: 0) + engine.mainMixerNode.installTap( + onBus: 0, + bufferSize: 4096, + format: engine.mainMixerNode.outputFormat(forBus: 0) + ) { [weak self] (buffer, time) in + guard let self = self else { return } + guard let recorder = self.recorder else { + self.engine.mainMixerNode.removeTap(onBus: 0) + return + } + + do { + let sampleBuffer = try Self.createAudioSampleBuffer(from: buffer, time: time) + recorder.audioInput.audioEngine(didOutputAudioSampleBuffer: sampleBuffer) + } + catch { + self.error = error + } + } } - } } - } - - let canDeactivateAudioSession: Bool - - @SCNObservable public internal(set) var error: Swift.Error? - - public init(canDeactivateAudioSession: Bool = true) { - self.canDeactivateAudioSession = canDeactivateAudioSession - setupObservers() - } - - deinit { - recorder?.audioInput.audioFormat = nil - player?.stop() - observers.forEach { notificationCenter.removeObserver($0) } - deactivateAudioSession() - } - - func activateAudioSession() throws { - try audioSession.setCategory(.playAndRecord, options: .defaultToSpeaker) - try audioSession.setActive(true) - } - - func deactivateAudioSession() { - guard canDeactivateAudioSession else { return } - - do { try audioSession.setActive(false) } - catch { self.error = error } - } + + let canDeactivateAudioSession: Bool + + @SCNObservable public internal(set) var error: Swift.Error? + + public init(canDeactivateAudioSession: Bool = true) { + self.canDeactivateAudioSession = canDeactivateAudioSession + setupObservers() + } + + deinit { + recorder?.audioInput.audioFormat = nil + player?.stop() + observers.forEach { notificationCenter.removeObserver($0) } + deactivateAudioSession() + } + + func activateAudioSession() throws { + try audioSession.setCategory(.playAndRecord, options: .defaultToSpeaker) + try audioSession.setActive(true) + } + + func deactivateAudioSession() { + guard canDeactivateAudioSession else { return } + + do { try audioSession.setActive(false) } + catch { self.error = error } + } } // MARK: - Observers @available(iOS 13.0, *) extension AudioEngine { - - func setupObservers() { - observers.append( - notificationCenter.addObserver( - forName: AVAudioSession.interruptionNotification, - object: nil, - queue: nil, - using: { [weak self] in self?.handleInterruption($0) } - ) - ) - - observers.append( - notificationCenter.addObserver( - forName: AVAudioSession.mediaServicesWereResetNotification, - object: nil, - queue: nil, - using: { [weak self] in self?.handleMediaServicesWereReset($0) } - ) - ) - - observers.append( - notificationCenter.addObserver( - forName: UIApplication.willResignActiveNotification, - object: nil, - queue: nil, - using: { [weak self] in self?.handleApplicationWillResignActiveNotification($0) } - ) - ) - - observers.append( - notificationCenter.addObserver( - forName: UIApplication.didBecomeActiveNotification, - object: nil, - queue: nil, - using: { [weak self] in self?.handleApplicationDidBecomeActive($0) }) - ) - } - - func handleApplicationWillResignActiveNotification(_ notification: Notification) { - state = .interrupted(playing: queue.sync { player?.state.isPlaying ?? false }) - player?.pause() - } - - func handleApplicationDidBecomeActive(_ notification: Notification) { - let shouldResume = state.shouldResume - state = .normal - if shouldResume { player?.play() } - } - - func handleInterruption(_ notification: Notification) { - guard let userInfo = notification.userInfo, - let typeValue = userInfo[AVAudioSessionInterruptionTypeKey] as? UInt, - let type = AVAudioSession.InterruptionType(rawValue: typeValue) - else { return } - - switch type { - case .began: - state = .interrupted(playing: queue.sync { player?.state.isPlaying ?? false }) - player?.pause() - - case .ended: - let shouldResume = state.shouldResume - state = .normal - - guard shouldResume, - let optionsValue = userInfo[AVAudioSessionInterruptionOptionKey] as? UInt, - AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume) - else { return } - player?.play() - - @unknown default: - break + + func setupObservers() { + observers.append( + notificationCenter.addObserver( + forName: AVAudioSession.interruptionNotification, + object: nil, + queue: nil, + using: { [weak self] in self?.handleInterruption($0) } + ) + ) + + observers.append( + notificationCenter.addObserver( + forName: AVAudioSession.mediaServicesWereResetNotification, + object: nil, + queue: nil, + using: { [weak self] in self?.handleMediaServicesWereReset($0) } + ) + ) + + observers.append( + notificationCenter.addObserver( + forName: UIApplication.willResignActiveNotification, + object: nil, + queue: nil, + using: { [weak self] in self?.handleApplicationWillResignActiveNotification($0) } + ) + ) + + observers.append( + notificationCenter.addObserver( + forName: UIApplication.didBecomeActiveNotification, + object: nil, + queue: nil, + using: { [weak self] in self?.handleApplicationDidBecomeActive($0) }) + ) + } + + func handleApplicationWillResignActiveNotification(_ notification: Notification) { + state = .interrupted(playing: queue.sync { player?.state.isPlaying ?? false }) + player?.pause() + } + + func handleApplicationDidBecomeActive(_ notification: Notification) { + let shouldResume = state.shouldResume + state = .normal + if shouldResume { player?.play() } + } + + func handleInterruption(_ notification: Notification) { + guard let userInfo = notification.userInfo, + let typeValue = userInfo[AVAudioSessionInterruptionTypeKey] as? UInt, + let type = AVAudioSession.InterruptionType(rawValue: typeValue) + else { return } + + switch type { + case .began: + state = .interrupted(playing: queue.sync { player?.state.isPlaying ?? false }) + player?.pause() + + case .ended: + let shouldResume = state.shouldResume + state = .normal + + guard shouldResume, + let optionsValue = userInfo[AVAudioSessionInterruptionOptionKey] as? UInt, + AVAudioSession.InterruptionOptions(rawValue: optionsValue).contains(.shouldResume) + else { return } + player?.play() + + @unknown default: + break + } + } + + func handleMediaServicesWereReset(_ notification: Notification) { + let player = self.player + let recorder = self.recorder + + self.player = nil + self.recorder = nil + + playerNode = AVAudioPlayerNode() + raterNode = AVAudioUnitTimePitch() + engine = AVAudioEngine() + engine.attach(playerNode) + engine.attach(raterNode) + + self.player = player + self.recorder = recorder } - } - - func handleMediaServicesWereReset(_ notification: Notification) { - let player = self.player - let recorder = self.recorder - - self.player = nil - self.recorder = nil - - playerNode = AVAudioPlayerNode() - raterNode = AVAudioUnitTimePitch() - engine = AVAudioEngine() - engine.attach(playerNode) - engine.attach(raterNode) - - self.player = player - self.recorder = recorder - } } @available(iOS 13.0, *) extension AudioEngine { - - static func createAudioSampleBuffer(from buffer: AVAudioPCMBuffer, time: AVAudioTime) throws -> CMSampleBuffer { - let audioBufferList = buffer.mutableAudioBufferList - let streamDescription = buffer.format.streamDescription.pointee - let timescale = CMTimeScale(streamDescription.mSampleRate) - let format = try CMAudioFormatDescription(audioStreamBasicDescription: streamDescription) - let sampleBuffer = try CMSampleBuffer( - dataBuffer: nil, - formatDescription: format, - numSamples: CMItemCount(buffer.frameLength), - sampleTimings: [ - CMSampleTimingInfo( - duration: CMTime(value: 1, timescale: timescale), - presentationTimeStamp: CMTime( - seconds: AVAudioTime.seconds(forHostTime: time.hostTime), - preferredTimescale: timescale - ), - decodeTimeStamp: .invalid + + static func createAudioSampleBuffer(from buffer: AVAudioPCMBuffer, time: AVAudioTime) throws -> CMSampleBuffer { + let audioBufferList = buffer.mutableAudioBufferList + let streamDescription = buffer.format.streamDescription.pointee + let timescale = CMTimeScale(streamDescription.mSampleRate) + let format = try CMAudioFormatDescription(audioStreamBasicDescription: streamDescription) + let sampleBuffer = try CMSampleBuffer( + dataBuffer: nil, + formatDescription: format, + numSamples: CMItemCount(buffer.frameLength), + sampleTimings: [ + CMSampleTimingInfo( + duration: CMTime(value: 1, timescale: timescale), + presentationTimeStamp: CMTime( + seconds: AVAudioTime.seconds(forHostTime: time.hostTime), + preferredTimescale: timescale + ), + decodeTimeStamp: .invalid + ) + ], + sampleSizes: [] ) - ], - sampleSizes: [] - ) - try sampleBuffer.setDataBuffer(fromAudioBufferList: audioBufferList) - return sampleBuffer - } + try sampleBuffer.setDataBuffer(fromAudioBufferList: audioBufferList) + return sampleBuffer + } }