swift - 如何裁剪和翻转 CVPixelBuffer 并返回 CVPixelBuffer?
问题描述
我正在制作一个快速的视频应用程序。
在我的应用程序中,我需要裁剪和水平翻转 CVPixelBuffer 并返回类型也是 CVPixelBuffer 的结果。
我尝试了几件事。
首先,我使用了“CVPixelBufferCreateWithBytes”
func resizePixelBuffer(_ pixelBuffer: CVPixelBuffer, destSize: CGSize)
-> CVPixelBuffer?
{
CVPixelBufferLockAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: O))
let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
var destPixelBuffer: CVPixelBuffer?
let topMargin = (height - destsize.height) / 2
let leftMargin = (width - destsize.width) / 2 * 4 // bytesPerPixel
let offset = topMargin * bytesPerRow + leftMargin
CVPixelBufferCreateWithBytes(kCFAllocatorDefault,
destSize.width,
destSize.height,
pixelFormat,
baseAddress.advanced(by: offset),
bytesPerRow,
nil, nil, nil,
&destPixelBuffer)
CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: O))
return destPixelBuffer
)
使用此代码,我可以直接裁剪 CVPixelBuffer 并返回 CVPixelBuffer。但是,我不知道如何水平翻转 CVPlxelBuffer。
所以我尝试了其他解决方案。
几秒钟后,我将 CVPixelBuffer 转换为 CIImage,然后返回 CVPixelBuffer
func resizePixelBuffer(_ pixelBuffer, destSize: CGSize)
-> CVPixelBuffer?
{
let bufferWidth = CVPixelBufferGetWidth(pixelBuffer)
let bufferHeight = CVPixelBufferGetHeight(pixelBuffer)
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
let rect = CGRect(x: (bufferWidth - destSize.width)/2, y: (bufferHeight - destSize.height)/2, width: destSize.width, height: destSize.height)
let croppedImage = ciImage.cropped(to: rect)
croppedImage.transformed(by: CGAffineTransform(translateX: -1, y: 0))
var destPixelBuffer: CVPixelBuffer?
CVPixelBufferCreate(kCFAllocatorDefault, destSize.width, destSize.height,
CVPixelBufferGetPixelFormatType(pixelBuffer), nil,
&destPixelBuffer)
CIContext().render(croppedImage, to: destPixelBuffer!, bounds: croppedImage.extent, croppedImage.colorSpace)
return destPixelBuffer
}
但结果并不如我所料。图像的某些部分是黑色的,我认为 CGAffineTransform 不起作用。
最后,我尝试转换为 CGImage
func resizePixelBuffer(_ pixelBuffer: CVPixelBuffer, destSize: CGSize)
-> CVPixelBuffer?
{
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
let cgImage = CIContext().createCGImage(ciImage, from: ciImage.extent)
let rect = CGRect(x: (bufferWidth - destSize.width)/2, y: (bufferHeight - destSize.height)/2, width: destSize.width, height: destSize.height)
let croppedImage = cgImage.cropping(to: rect)
let width = croppedImage.width
let height = croppedImage.height
let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
var destPixelBuffer: CVPixelBuffer?
CVPixelBufferCreate(kCFAllocatorDefault, width, height, pixelFormat, &destPixelBuffer)
CVPixelBufferLockBaseAddress(destPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
let destBaseAddress = CVPixelBufferGetBaseAddress(destPixelBuffer)
let destBytesPerRow = CVPixelBufferGetBytesPerRow(destPixelBuffer)
let context = CGContext(data: destBaseAddress,
width: width,
height: height,
bitsPerComponent: 8,
bytesPerRow: destBytesPerRow,
space: croppedImage.colorSpace,
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)
context?.concatenate(__CGAffineTransformMake( 1, 0, 0, -1, 0, CGFloat(height)))
context?.draw(croppedCgImage, in: CGRect(x: 0, y: 0, width: CGFloat(width), height: CGFloat(height)))
CVPixelBufferUnlockBaseAddress(srcPixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
return destPixelBuffer
}
此时,输出像素缓冲区是全黑的。
我不知道如何裁剪和翻转 CVPixelBuffer 并返回 CVPixelBuffer。
我认为转换 CIImage 或 CGImage 是更好的方法,因为我可以用这些格式做很多事情。
但我不知道如何将这些格式转换回 CVPixelBuffer。
请让我知道如何做到这一点。
解决方案
这是CVPixelBuffer
使用中的一个流程vImage
。
- 检查iOS - 缩放和裁剪 CMSampleBufferRef/CVImageBufferRef以了解裁剪对缓冲区的作用
- 检查https://developer.apple.com/documentation/accelerate/reading_from_and_writing_to_core_video_pixel_buffers了解 vImage 的基本用法。基本上,您需要创建源缓冲区和目标缓冲区并对其应用转换。
- vImageBuffer 到 CVPixelBuffer 部分引用自https://github.com/tensorflow/examples/blob/master/lite/examples/object_detection/ios/ObjectDetection/Extensions/CVPixelBufferExtension.swift
import Cocoa
import Accelerate
extension CVPixelBuffer {
func crop(to rect: CGRect) -> CVPixelBuffer? {
CVPixelBufferLockBaseAddress(self, .readOnly)
defer { CVPixelBufferUnlockBaseAddress(self, .readOnly) }
guard let baseAddress = CVPixelBufferGetBaseAddress(self) else {
return nil
}
let inputImageRowBytes = CVPixelBufferGetBytesPerRow(self)
let imageChannels = 4
let startPos = Int(rect.origin.y) * inputImageRowBytes + imageChannels * Int(rect.origin.x)
let outWidth = UInt(rect.width)
let outHeight = UInt(rect.height)
let croppedImageRowBytes = Int(outWidth) * imageChannels
var inBuffer = vImage_Buffer()
inBuffer.height = outHeight
inBuffer.width = outWidth
inBuffer.rowBytes = inputImageRowBytes
inBuffer.data = baseAddress + UnsafeMutableRawPointer.Stride(startPos)
guard let croppedImageBytes = malloc(Int(outHeight) * croppedImageRowBytes) else {
return nil
}
var outBuffer = vImage_Buffer(data: croppedImageBytes, height: outHeight, width: outWidth, rowBytes: croppedImageRowBytes)
let scaleError = vImageScale_ARGB8888(&inBuffer, &outBuffer, nil, vImage_Flags(0))
guard scaleError == kvImageNoError else {
free(croppedImageBytes)
return nil
}
return croppedImageBytes.toCVPixelBuffer(pixelBuffer: self, targetWith: Int(outWidth), targetHeight: Int(outHeight), targetImageRowBytes: croppedImageRowBytes)
}
func flip() -> CVPixelBuffer? {
CVPixelBufferLockBaseAddress(self, .readOnly)
defer { CVPixelBufferUnlockBaseAddress(self, .readOnly) }
guard let baseAddress = CVPixelBufferGetBaseAddress(self) else {
return nil
}
let width = UInt(CVPixelBufferGetWidth(self))
let height = UInt(CVPixelBufferGetHeight(self))
let inputImageRowBytes = CVPixelBufferGetBytesPerRow(self)
let outputImageRowBytes = inputImageRowBytes
var inBuffer = vImage_Buffer(
data: baseAddress,
height: height,
width: width,
rowBytes: inputImageRowBytes)
guard let targetImageBytes = malloc(Int(height) * outputImageRowBytes) else {
return nil
}
var outBuffer = vImage_Buffer(data: targetImageBytes, height: height, width: width, rowBytes: outputImageRowBytes)
// See https://developer.apple.com/documentation/accelerate/vimage/vimage_operations/image_reflection for other transformations
let reflectError = vImageHorizontalReflect_ARGB8888(&inBuffer, &outBuffer, vImage_Flags(0))
// let reflectError = vImageVerticalReflect_ARGB8888(&inBuffer, &outBuffer, vImage_Flags(0))
guard reflectError == kvImageNoError else {
free(targetImageBytes)
return nil
}
return targetImageBytes.toCVPixelBuffer(pixelBuffer: self, targetWith: Int(width), targetHeight: Int(height), targetImageRowBytes: outputImageRowBytes)
}
}
extension UnsafeMutableRawPointer {
// Converts the vImage buffer to CVPixelBuffer
func toCVPixelBuffer(pixelBuffer: CVPixelBuffer, targetWith: Int, targetHeight: Int, targetImageRowBytes: Int) -> CVPixelBuffer? {
let pixelBufferType = CVPixelBufferGetPixelFormatType(pixelBuffer)
let releaseCallBack: CVPixelBufferReleaseBytesCallback = {mutablePointer, pointer in
if let pointer = pointer {
free(UnsafeMutableRawPointer(mutating: pointer))
}
}
var targetPixelBuffer: CVPixelBuffer?
let conversionStatus = CVPixelBufferCreateWithBytes(nil, targetWith, targetHeight, pixelBufferType, self, targetImageRowBytes, releaseCallBack, nil, nil, &targetPixelBuffer)
guard conversionStatus == kCVReturnSuccess else {
free(self)
return nil
}
return targetPixelBuffer
}
}
// Change this to your input pixelBuffer
var pixelBuffer: CVPixelBuffer?
// The result would be stored in resultPixelBuffer
let resultPixelBuffer = pixelBuffer?.crop(to: CGRect(x: 50, y: 50, width: 100, height: 100))?.flip()
推荐阅读
- python - 如何更改我的代码中的 for 循环以在我的数据框中为我提供一个额外的列?
- python-3.x - 有什么区别?
- azure-pipelines - Azure DevOps 上的新管道显示错误消息“检测到与管道触发器相关的一些最新问题”。
- spring - 从 URL 中读取日语查询参数
- node.js - 出现错误:在为 webdriverio 执行 npm install 后,码支持最低 Node.js 版本 12
- reactjs - 我的 api 上的 React.js fetch 方法不起作用?
- typescript - 替换/引用字符串常量时出错
- swift - 如何在 SwiftUI 中使用 Core Data 保存文本和清单项?
- python - 如何连接具有多对多关系的 Django 模型?
- sonarqube - SonarQube - 添加自定义质量门