ios - 通过 iOS 中的 AVAssetReaderTrackOutput 获取样本缓冲区时音频丢失?
问题描述
在这里,我使用资产阅读器获取样本缓冲区,然后处理每个帧以用于自定义目的。但是保存在文档中的最终视频缺少音频。我知道还有另一种处理每个帧的方法,例如“applyingCIFiltersWithHandler”,但需要每个样本缓冲区并渲染图像或过滤它。建议我解决这个问题?
NSError *error;
NSString *path = [[NSBundle mainBundle] pathForResource:@"recordmovie" ofType:@"mov"];
NSURL *videoURL = [NSURL fileURLWithPath:path];
AVURLAsset *asset = [AVURLAsset URLAssetWithURL:videoURL options:nil];;
AVAssetReader *reader = [[AVAssetReader alloc] initWithAsset:asset error:nil];
AVAssetTrack *videoTrack = [[asset tracksWithMediaType:AVMediaTypeVideo] firstObject];
// add audio track here
AVAssetTrack *audioTrack = [[asset tracksWithMediaType:AVMediaTypeAudio] firstObject];
NSDictionary *readerOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange], kCVPixelBufferPixelFormatTypeKey, nil];
CGSize renderSize = [videoTrack naturalSize];
/*
NSDictionary *readerOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
AVVideoCodecH264 , AVVideoCodecKey,
renderSize.width , AVVideoWidthKey,
renderSize.height , AVVideoHeightKey,
AVVideoScalingModeResizeAspectFill,AVVideoScalingModeKey, nil];
*/
AVAssetReaderTrackOutput* readerOutput = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:videoTrack
outputSettings:readerOutputSettings];
AudioChannelLayout acl;
bzero( &acl, sizeof(acl));
acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;
NSDictionary* audioOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
[ NSNumber numberWithInt: kAudioFormatMPEG4AAC], AVFormatIDKey,
[ NSNumber numberWithInt: 1 ], AVNumberOfChannelsKey,
[ NSNumber numberWithFloat: 44100.0 ], AVSampleRateKey,
[ NSData dataWithBytes: &acl length: sizeof( acl ) ], AVChannelLayoutKey,
[ NSNumber numberWithInt: 64000 ], AVEncoderBitRateKey,
nil];
NSDictionary *settings = @{ AVFormatIDKey : [NSNumber numberWithInt:kAudioFormatLinearPCM] };
AVAssetReaderTrackOutput *audioTrackOutput = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:audioTrack outputSettings:settings];
[reader addOutput:readerOutput];
[reader addOutput:audioTrackOutput];
[reader startReading];
NSMutableArray *samples = [[NSMutableArray alloc] init];
CMSampleBufferRef sample;
while((sample = [readerOutput copyNextSampleBuffer])) {
[samples addObject:(__bridge id)sample];
CFRelease(sample);
}
NSString *outputPath = [self getDocumentsUrlForFilterMovie];
NSURL *outputURL = [NSURL fileURLWithPath:outputPath];
AVAssetWriter *writer = [[AVAssetWriter alloc] initWithURL:outputURL
fileType:AVFileTypeQuickTimeMovie
error:&error];
NSDictionary *videoCompressionProps = [NSDictionary dictionaryWithObjectsAndKeys:
@(videoTrack.estimatedDataRate), AVVideoAverageBitRateKey,
nil];
NSDictionary *writerOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
AVVideoCodecH264, AVVideoCodecKey,
[NSNumber numberWithInt:videoTrack.naturalSize.width], AVVideoWidthKey,
[NSNumber numberWithInt:videoTrack.naturalSize.height], AVVideoHeightKey,
videoCompressionProps, AVVideoCompressionPropertiesKey,
nil];
AVAssetWriterInput *writerInput = [[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo
outputSettings:writerOutputSettings
sourceFormatHint:(__bridge CMFormatDescriptionRef)[videoTrack.formatDescriptions lastObject]];
[writerInput setExpectsMediaDataInRealTime:NO];
[writer addInput:writerInput];
AVAssetWriterInput *WriterAudioInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio outputSettings:audioOutputSettings];
WriterAudioInput.expectsMediaDataInRealTime = YES;
if([writer canAddInput:WriterAudioInput]) {
[writer addInput:WriterAudioInput];
}
AVAssetWriterInputPixelBufferAdaptor *pixelBufferAdaptor = [[AVAssetWriterInputPixelBufferAdaptor alloc] initWithAssetWriterInput:writerInput sourcePixelBufferAttributes:nil];
[writer startWriting];
[writer startSessionAtSourceTime:CMSampleBufferGetPresentationTimeStamp((__bridge CMSampleBufferRef)samples[0])];
//NSMutableArray *audioSamples = [[NSMutableArray alloc] init];
while((sample = [audioTrackOutput copyNextSampleBuffer])) {
//[audioSamples addObject:(__bridge id)sample];
[WriterAudioInput appendSampleBuffer:sample];
while (!WriterAudioInput.readyForMoreMediaData) {
[NSThread sleepForTimeInterval:0.1];
}
CFRelease(sample);
}
CIFilter *filter = [CIFilter filterWithName:@"CISepiaTone"];
[filter setDefaults];
[filter setValue:@(1) forKey:kCIInputIntensityKey];
//CIImage *outputImage = filter.outputImage;
for(NSInteger i = 0; i < samples.count; i++) {
CMTime presentationTime = CMSampleBufferGetPresentationTimeStamp((__bridge CMSampleBufferRef)samples[i]);
//CVPixelBufferRef videoFrameBuffer = CMSampleBufferGetImageBuffer((__bridge CMSampleBufferRef)samples[samples.count - i - 1]);
CVPixelBufferRef videoFrameBuffer = CMSampleBufferGetImageBuffer((__bridge CMSampleBufferRef)samples[i]);
CIImage *frameImage = [CIImage imageWithCVPixelBuffer:videoFrameBuffer];
[filter setValue:frameImage forKey:kCIInputImageKey];
CIImage *outputImage = filter.outputImage;
//}
[self->ciContext render:outputImage toCVPixelBuffer:videoFrameBuffer bounds:outputImage.extent colorSpace:self->colorSpace];
while (!writerInput.readyForMoreMediaData) {
[NSThread sleepForTimeInterval:0.1];
}
// [writerInput appendSampleBuffer:videoFrameBuffer];
[pixelBufferAdaptor appendPixelBuffer:videoFrameBuffer withPresentationTime:presentationTime];
}
[writerInput markAsFinished];
[writer finishWritingWithCompletionHandler:^(){
//[self.delegate didFinishReverse:YES andVideoURL:outputURL withError:error];
NSLog(@"Finish video rendering");
}];
});
解决方案
错过将音频文件添加到 audioAssetWriterInput。我通过添加音频样本缓冲区解决了这个问题。
在这里,我添加代码以从现有视频中获取音频、视频样本缓冲区,然后写入并保存到本地文档。您可以为特定帧区域中的所需帧和总帧应用过滤器并在图像上渲染图像。
NSError *error;
AVURLAsset *videoAsset = [AVURLAsset URLAssetWithURL:videoURL options:nil];;
AVAssetReader *assetReader = [[AVAssetReader alloc] initWithAsset:videoAsset error:nil];
AVAssetTrack *videoTrack = [[videoAsset tracksWithMediaType:AVMediaTypeVideo] firstObject];
AVAssetTrack *audioTrack = [[videoAsset tracksWithMediaType:AVMediaTypeAudio] firstObject];
NSDictionary *videoReaderOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange], kCVPixelBufferPixelFormatTypeKey, nil];
AVAssetReaderTrackOutput* assetReaderVideoTrackOutput = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:videoTrack outputSettings:videoReaderOutputSettings];
AudioChannelLayout acl;
bzero( &acl, sizeof(acl));
acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;
NSDictionary* audioOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
[ NSNumber numberWithInt: kAudioFormatMPEG4AAC], AVFormatIDKey,
[ NSNumber numberWithInt: 1 ], AVNumberOfChannelsKey,
[ NSNumber numberWithFloat: 44100.0 ], AVSampleRateKey,
[ NSData dataWithBytes: &acl length: sizeof( acl ) ], AVChannelLayoutKey,
[ NSNumber numberWithInt: 64000 ], AVEncoderBitRateKey,
nil];
NSDictionary *audioDecodesettings = @{ AVFormatIDKey : [NSNumber numberWithInt:kAudioFormatLinearPCM] };
AVAssetReaderTrackOutput *assetReaderAudioTrackOutput = [AVAssetReaderTrackOutput assetReaderTrackOutputWithTrack:audioTrack outputSettings:audioDecodesettings];
[assetReader addOutput:assetReaderVideoTrackOutput];
[assetReader addOutput:assetReaderAudioTrackOutput];
[assetReader startReading];
NSMutableArray *samples = [[NSMutableArray alloc] init];
CMSampleBufferRef sample;
while((sample = [assetReaderVideoTrackOutput copyNextSampleBuffer])) {
[samples addObject:(__bridge id)sample];
CFRelease(sample);
}
NSString *outputPath = [self getDocumentsUrlForFilterMovie];
NSURL *outputURL = [NSURL fileURLWithPath:outputPath];
AVAssetWriter *assetWriter = [[AVAssetWriter alloc] initWithURL:outputURL
fileType:AVFileTypeQuickTimeMovie
error:&error];
NSDictionary *videoCompressionProps = [NSDictionary dictionaryWithObjectsAndKeys:
@(videoTrack.estimatedDataRate), AVVideoAverageBitRateKey,
nil];
NSDictionary *writerOutputSettings = [NSDictionary dictionaryWithObjectsAndKeys:
AVVideoCodecH264, AVVideoCodecKey,
[NSNumber numberWithInt:videoTrack.naturalSize.width], AVVideoWidthKey,
[NSNumber numberWithInt:videoTrack.naturalSize.height], AVVideoHeightKey,
videoCompressionProps, AVVideoCompressionPropertiesKey,
nil];
AVAssetWriterInput *videoWriterInput = [[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo
outputSettings:writerOutputSettings
sourceFormatHint:(__bridge CMFormatDescriptionRef)[videoTrack.formatDescriptions lastObject]];
[videoWriterInput setExpectsMediaDataInRealTime:NO];
[assetWriter addInput:videoWriterInput];
AVAssetWriterInput *audioWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeAudio outputSettings:audioOutputSettings];
audioWriterInput.expectsMediaDataInRealTime = YES;
if([assetWriter canAddInput:audioWriterInput]) {
[assetWriter addInput:audioWriterInput];
}
AVAssetWriterInputPixelBufferAdaptor *pixelBufferAdaptor = [[AVAssetWriterInputPixelBufferAdaptor alloc] initWithAssetWriterInput:videoWriterInput sourcePixelBufferAttributes:nil];
[assetWriter startWriting];
[assetWriter startSessionAtSourceTime:CMSampleBufferGetPresentationTimeStamp((__bridge CMSampleBufferRef)samples[0])];
while((sample = [assetReaderAudioTrackOutput copyNextSampleBuffer])) {
[audioWriterInput appendSampleBuffer:sample];
while (!audioWriterInput.readyForMoreMediaData) {
[NSThread sleepForTimeInterval:0.1];
}
CFRelease(sample);
}
for(NSInteger i = 0; i < samples.count; i++) {
CMTime presentationTime = CMSampleBufferGetPresentationTimeStamp((__bridge CMSampleBufferRef)samples[i]);
CVPixelBufferRef videoFrameBuffer = nil;
if(frameRenderType == KVideoNormal) {
videoFrameBuffer = CMSampleBufferGetImageBuffer((__bridge CMSampleBufferRef)samples[i]);
} else if (frameRenderType == KVideoReverse) {
videoFrameBuffer = CMSampleBufferGetImageBuffer((__bridge CMSampleBufferRef)samples[samples.count - i - 1]);
}
if(self.filters.count > 0) {
CIImage *frameImage = [CIImage imageWithCVPixelBuffer:videoFrameBuffer];
for(CIFilter *filter in self.filters) {
[filter setValue:frameImage forKey:kCIInputImageKey];
frameImage = filter.outputImage;
}
[self->ciContext render:frameImage toCVPixelBuffer:videoFrameBuffer bounds:frameImage.extent colorSpace:self->colorSpace];
}
while (!videoWriterInput.readyForMoreMediaData) {
[NSThread sleepForTimeInterval:0.1];
}
[pixelBufferAdaptor appendPixelBuffer:videoFrameBuffer withPresentationTime:presentationTime];
}
[videoWriterInput markAsFinished];
[assetWriter finishWritingWithCompletionHandler:^(){
dispatch_async(dispatch_get_main_queue(), ^{
NSLog(@"Finished video processing");
});
}];
});
推荐阅读
- python-3.x - 如何让 sublime text 3 编辑器识别 python 3?
- flutter - 无法在“floatingActionButton”小部件上禁用外观动画
- android - 原因:无效类型代码:69 Android studio while building
- javascript - 如何使用依赖项触发 UseEffect Once?
- python - 有人可以向我解释 MinMaxScaler() 的工作原理吗?
- reactjs - React组件:视频运行,但流不通过方法运行
- view - 结合来自 Google Analytics 的 View 数据
- python - 多个线程函数修改同一个列表
- python - 在 Python 中使用 re.search(pattern, text) 提取两个指定子字符串之间的子字符串
- android - 未解决的参考:在 lateinit 属性上的 isInitialized