ios - MTAudioProcessingTap EXC_BAD_ACCESS ，并不总是触发 finalize 回调。如何释放它？
问题描述
我正在尝试实现MTAudioProcessingTap并且效果很好。问题是当我完成使用 Tap 并重新实例化我的类并创建一个新的 Tap 时。
我假设如何释放水龙头 1- 我在创建时将水龙头保留为属性，希望我可以访问它并稍后释放它 2- 在类的deinit()方法中，我将 audiomix 设置为 nil 并尝试做一个自我.tap?.release()
问题是..有时它可以工作并调用 FINALIZE 回调，一切都很好，有时它不会，只是在tapProcess Callback 行崩溃：
let selfMediaInput = Unmanaged<VideoMediaInput>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).takeUnretainedValue()
这是完整的代码：https ://gist.github.com/omarojo/03d08165a1a7962cb30c17ec01f809a3
import Foundation
import UIKit
import AVFoundation;
import MediaToolbox
protocol VideoMediaInputDelegate: class {
func videoFrameRefresh(sampleBuffer: CMSampleBuffer) //could be audio or video
}
class VideoMediaInput: NSObject {
private let queue = DispatchQueue(label: "com.GenerateMetal.VideoMediaInput")
var videoURL: URL!
weak var delegate: VideoMediaInputDelegate?
private var playerItemObserver: NSKeyValueObservation?
var displayLink: CADisplayLink!
var player = AVPlayer()
var playerItem: AVPlayerItem!
let videoOutput = AVPlayerItemVideoOutput(pixelBufferAttributes: [String(kCVPixelBufferPixelFormatTypeKey): NSNumber(value: kCVPixelFormatType_32BGRA)])
var audioProcessingFormat: AudioStreamBasicDescription?//UnsafePointer<AudioStreamBasicDescription>?
var tap: Unmanaged<MTAudioProcessingTap>?
override init(){
}
convenience init(url: URL){
self.init()
self.videoURL = url
self.playerItem = AVPlayerItem(url: url)
playerItemObserver = playerItem.observe(\.status) { [weak self] item, _ in
guard item.status == .readyToPlay else { return }
self?.playerItemObserver = nil
self?.player.play()
}
setupProcessingTap()
player.replaceCurrentItem(with: playerItem)
player.currentItem!.add(videoOutput)
NotificationCenter.default.removeObserver(self)
NotificationCenter.default.addObserver(forName: NSNotification.Name.AVPlayerItemDidPlayToEndTime, object: nil, queue: nil) {[weak self] notification in
if let weakSelf = self {
/*
Setting actionAtItemEnd to None prevents the movie from getting paused at item end. A very simplistic, and not gapless, looped playback.
*/
weakSelf.player.actionAtItemEnd = .none
weakSelf.player.seek(to: CMTime.zero)
weakSelf.player.play()
}
}
NotificationCenter.default.addObserver(
self,
selector: #selector(applicationDidBecomeActive(_:)),
name: UIApplication.didBecomeActiveNotification,
object: nil)
}
func stopAllProcesses(){
self.queue.sync {
self.player.pause()
self.player.isMuted = true
self.player.currentItem?.audioMix = nil
self.playerItem.audioMix = nil
self.playerItem = nil
self.tap?.release()
}
}
deinit{
print(">> VideoInput deinited !!!! ")
if let link = self.displayLink {
link.invalidate()
}
NotificationCenter.default.removeObserver(self)
stopAllProcesses()
}
public func playVideo(){
if (player.currentItem != nil) {
print("Starting playback!")
player.play()
}
}
public func pauseVideo(){
if (player.currentItem != nil) {
print("Pausing playback!")
player.pause()
}
}
@objc func applicationDidBecomeActive(_ notification: NSNotification) {
playVideo()
}
//MARK: GET AUDIO BUFFERS
func setupProcessingTap(){
var callbacks = MTAudioProcessingTapCallbacks(
version: kMTAudioProcessingTapCallbacksVersion_0,
clientInfo: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()),
init: tapInit,
finalize: tapFinalize,
prepare: tapPrepare,
unprepare: tapUnprepare,
process: tapProcess)
var tap: Unmanaged<MTAudioProcessingTap>?
let err = MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks, kMTAudioProcessingTapCreationFlag_PostEffects, &tap)
self.tap = tap
print("err: \(err)\n")
if err == noErr {
}
print("tracks? \(playerItem.asset.tracks)\n")
let audioTrack = playerItem.asset.tracks(withMediaType: AVMediaType.audio).first!
let inputParams = AVMutableAudioMixInputParameters(track: audioTrack)
inputParams.audioTapProcessor = tap?.takeRetainedValue()//tap?.takeUnretainedValue()
// tap?.release()
// print("inputParms: \(inputParams), \(inputParams.audioTapProcessor)\n")
let audioMix = AVMutableAudioMix()
audioMix.inputParameters = [inputParams]
playerItem.audioMix = audioMix
}
//MARK: TAP CALLBACKS
let tapInit: MTAudioProcessingTapInitCallback = {
(tap, clientInfo, tapStorageOut) in
tapStorageOut.pointee = clientInfo
print("init \(tap, clientInfo, tapStorageOut)\n")
}
let tapFinalize: MTAudioProcessingTapFinalizeCallback = {
(tap) in
print("finalize \(tap)\n")
}
let tapPrepare: MTAudioProcessingTapPrepareCallback = {
(tap, itemCount, basicDescription) in
print("prepare: \(tap, itemCount, basicDescription)\n")
let selfMediaInput = Unmanaged<VideoMediaInput>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).takeUnretainedValue()
selfMediaInput.audioProcessingFormat = AudioStreamBasicDescription(mSampleRate: basicDescription.pointee.mSampleRate,
mFormatID: basicDescription.pointee.mFormatID, mFormatFlags: basicDescription.pointee.mFormatFlags, mBytesPerPacket: basicDescription.pointee.mBytesPerPacket, mFramesPerPacket: basicDescription.pointee.mFramesPerPacket, mBytesPerFrame: basicDescription.pointee.mBytesPerFrame, mChannelsPerFrame: basicDescription.pointee.mChannelsPerFrame, mBitsPerChannel: basicDescription.pointee.mBitsPerChannel, mReserved: basicDescription.pointee.mReserved)
}
let tapUnprepare: MTAudioProcessingTapUnprepareCallback = {
(tap) in
print("unprepare \(tap)\n")
}
let tapProcess: MTAudioProcessingTapProcessCallback = {
(tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) in
print("callback \(bufferListInOut)\n")
let selfMediaInput = Unmanaged<VideoMediaInput>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).takeUnretainedValue()
let status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, flagsOut, nil, numberFramesOut)
//print("get audio: \(status)\n")
if status != noErr {
print("Error TAPGetSourceAudio :\(String(describing: status.description))")
return
}
selfMediaInput.processAudioData(audioData: bufferListInOut, framesNumber: UInt32(numberFrames))
}
func processAudioData(audioData: UnsafeMutablePointer<AudioBufferList>, framesNumber: UInt32) {
var sbuf: CMSampleBuffer?
var status : OSStatus?
var format: CMFormatDescription?
//FORMAT
// var audioFormat = self.audioProcessingFormat//self.audioProcessingFormat?.pointee
guard var audioFormat = self.audioProcessingFormat else {
return
}
status = CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &audioFormat, layoutSize: 0, layout: nil, magicCookieSize: 0, magicCookie: nil, extensions: nil, formatDescriptionOut: &format)
if status != noErr {
print("Error CMAudioFormatDescriptionCreater :\(String(describing: status?.description))")
return
}
print(">> Audio Buffer mSampleRate:\(Int32(audioFormat.mSampleRate))")
var timing = CMSampleTimingInfo(duration: CMTimeMake(value: 1, timescale: Int32(audioFormat.mSampleRate)), presentationTimeStamp: self.player.currentTime(), decodeTimeStamp: CMTime.invalid)
status = CMSampleBufferCreate(allocator: kCFAllocatorDefault,
dataBuffer: nil,
dataReady: Bool(truncating: 0),
makeDataReadyCallback: nil,
refcon: nil,
formatDescription: format,
sampleCount: CMItemCount(framesNumber),
sampleTimingEntryCount: 1,
sampleTimingArray: &timing,
sampleSizeEntryCount: 0, sampleSizeArray: nil,
sampleBufferOut: &sbuf);
if status != noErr {
print("Error CMSampleBufferCreate :\(String(describing: status?.description))")
return
}
status = CMSampleBufferSetDataBufferFromAudioBufferList(sbuf!,
blockBufferAllocator: kCFAllocatorDefault ,
blockBufferMemoryAllocator: kCFAllocatorDefault,
flags: 0,
bufferList: audioData)
if status != noErr {
print("Error cCMSampleBufferSetDataBufferFromAudioBufferList :\(String(describing: status?.description))")
return
}
let currentSampleTime = CMSampleBufferGetOutputPresentationTimeStamp(sbuf!);
print(" audio buffer at time: \(currentSampleTime)")
self.delegate?.videoFrameRefresh(sampleBuffer: sbuf!)
}
}
我如何使用我的班级
self.inputVideoMedia = nil
self.inputVideoMedia = VideoMediaInput(url: videoURL)
self.inputVideoMedia!.delegate = self
我第二次这样做......它崩溃了（但并非总是如此）。它没有崩溃的时间我可以在控制台中看到 FINALIZE 打印。
解决方案
如果在点击被解除VideoMediaInput分配之前被解除分配（这可能会发生，因为似乎没有办法同步停止点击），那么点击回调可能会阻塞对已解除分配类的引用。
您可以通过传递（我猜是包装的）对您的类的弱引用来解决此问题。你可以这样做：
首先删除您的tap实例变量，以及对它的任何引用 - 它不是必需的。然后进行以下更改：
class VideoMediaInput: NSObject {
class TapCookie {
weak var input: VideoMediaInput?
deinit {
print("TapCookie deinit")
}
}
...
func setupProcessingTap(){
let cookie = TapCookie()
cookie.input = self
var callbacks = MTAudioProcessingTapCallbacks(
version: kMTAudioProcessingTapCallbacksVersion_0,
clientInfo: UnsafeMutableRawPointer(Unmanaged.passRetained(cookie).toOpaque()),
init: tapInit,
finalize: tapFinalize,
prepare: tapPrepare,
unprepare: tapUnprepare,
process: tapProcess)
...
let tapFinalize: MTAudioProcessingTapFinalizeCallback = {
(tap) in
print("finalize \(tap)\n")
// release cookie
Unmanaged<TapCookie>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).release()
}
let tapPrepare: MTAudioProcessingTapPrepareCallback = {
(tap, itemCount, basicDescription) in
print("prepare: \(tap, itemCount, basicDescription)\n")
let cookie = Unmanaged<TapCookie>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).takeUnretainedValue()
let selfMediaInput = cookie.input!
...
let tapProcess: MTAudioProcessingTapProcessCallback = {
(tap, numberFrames, flags, bufferListInOut, numberFramesOut, flagsOut) in
print("callback \(bufferListInOut)\n")
let cookie = Unmanaged<TapCookie>.fromOpaque(MTAudioProcessingTapGetStorage(tap)).takeUnretainedValue()
guard let selfMediaInput = cookie.input else {
print("Tap callback: VideoMediaInput was deallocated!")
return
}
...
我不确定是否需要 cookie 类，它的存在只是为了包装weak参考。尖端的 Swift 专家可能知道如何通过所有青少年突变忍者的原始指针来弥补弱点，但我不知道。
推荐阅读
- flutter - 修改默认的 ExpansionTile 子项垂直填充 Flutter
- mysql - 转移开始在我的 ASP.NET API 上花费的时间太长
- https - 为 macOS [mac os catalina 10.15.2] 在 localhost 上设置 HTTPS
- javascript - 如何设置 WordPress 和 WooCommerce API，以便在发布时不会出现 CORS 错误？
- autodesk-forge - 使用 Auto-desk Forge 实现登录系统
- python - 使用 pymongo 将数据插入 mongo 时删除 NaT 值
- c - 我的插座游戏有一些问题，我不明白为什么
- debian - Snort 规则配置问题
- python - 如何使用 BeautifulSoup 通过类似的 HTML 块“循环”？
- html - 推文后重定向 - 通过网站的 twitter 实现共享