import Foundation import VideoToolbox import AVFoundation /// 视频解码器,基于VideoToolbox实现H264/H265硬件解码,输出CVPixelBuffer class VideoDecoder { enum CodecType: String { case h264 = "h264" case h265 = "h265" var codecType: CMVideoCodecType { switch self { case .h264: return kCMVideoCodecType_H264 case .h265: return kCMVideoCodecType_HEVC } } } // ====== 关键成员变量注释 ====== /// 解码会话对象 private var decompressionSession: VTDecompressionSession? /// 视频格式描述 private var formatDesc: CMVideoFormatDescription? /// 视频宽度 private let width: Int /// 视频高度 private let height: Int /// 编码类型(H264/H265) private let codecType: CodecType /// 解码线程队列 private let decodeQueue = DispatchQueue(label: "video_decode_plugin.decode.queue") /// 解码会话是否已准备好 private var isSessionReady = false /// 最近一次I帧序号 private var lastIFrameSeq: Int? /// 已处理帧序号集合 private var frameSeqSet = Set() /// 最大允许延迟(毫秒) private let maxAllowedDelayMs: Int = 350 /// 时间戳基准 private var timestampBaseMs: Int64? /// 首帧相对时间戳 private var firstFrameRelativeTimestamp: Int64? // ====== 新增:缓冲区与自适应帧率相关成员 ====== /// 输入缓冲区(待解码帧队列),线程安全 private let inputQueue = DispatchQueue(label: "video_decode_plugin.input.queue", attributes: .concurrent) private var inputBuffer: [(frameData: Data, frameType: Int, timestamp: Int64, frameSeq: Int, refIFrameSeq: Int?, sps: Data?, pps: Data?)] = [] private let inputBufferSemaphore = DispatchSemaphore(value: 1) private let inputBufferMaxCount = 15 /// 输出缓冲区(解码后帧队列),线程安全 private let outputQueue = DispatchQueue(label: "video_decode_plugin.output.queue", attributes: .concurrent) private var outputBuffer: [(pixelBuffer: CVPixelBuffer, timestamp: Int64)] = [] private let outputBufferSemaphore = DispatchSemaphore(value: 1) private let outputBufferMaxCount = 15 /// 渲染线程 private var renderThread: Thread? /// 渲染线程运行标志 private var renderThreadRunning = false /// 首次渲染回调标志 private var hasNotifiedFlutter = false /// 当前渲染帧率 private var renderFps: Int = 15 /// EMA平滑后的帧率 private var smoothedFps: Double = 15.0 /// EMA平滑系数 private let alpha: Double = 0.2 /// 最小帧率 private let minFps: Double = 8.0 /// 最大帧率 private let maxFps: Double = 30.0 /// 单次最大调整幅度 private let maxStep: Double = 2.0 /// 渲染帧时间戳队列 private var renderedTimestamps: [Int64] = [] // ms /// 渲染帧时间戳最大数量 private let renderedTimestampsMaxCount = 20 /// 已渲染帧计数 private var renderedFrameCount = 0 /// 每N帧调整一次帧率 private let fpsAdjustInterval = 10 /// 解码回调,输出CVPixelBuffer和时间戳 var onFrameDecoded: ((CVPixelBuffer, Int64) -> Void)? = { _, _ in } /// 初始化解码器,启动渲染线程 init(width: Int, height: Int, codecType: String) { self.width = width self.height = height self.codecType = CodecType(rawValue: codecType.lowercased()) ?? .h264 startRenderThread() } // ====== 输入缓冲区操作 ====== /// 入队待解码帧 private func enqueueInput(_ item: (Data, Int, Int64, Int, Int?, Data?, Data?)) { inputQueue.async(flags: .barrier) { if self.inputBuffer.count >= self.inputBufferMaxCount { self.inputBuffer.removeFirst() } self.inputBuffer.append(item) } } /// 出队待解码帧 private func dequeueInput() -> (Data, Int, Int64, Int, Int?, Data?, Data?)? { var item: (Data, Int, Int64, Int, Int?, Data?, Data?)? inputQueue.sync { if !self.inputBuffer.isEmpty { item = self.inputBuffer.removeFirst() } } return item } // ====== 输出缓冲区操作 ====== /// 入队解码后帧 private func enqueueOutput(_ item: (CVPixelBuffer, Int64)) { outputQueue.async(flags: .barrier) { if self.outputBuffer.count >= self.outputBufferMaxCount { self.outputBuffer.removeFirst() } self.outputBuffer.append(item) } } /// 出队解码后帧 private func dequeueOutput() -> (CVPixelBuffer, Int64)? { var item: (CVPixelBuffer, Int64)? outputQueue.sync { if !self.outputBuffer.isEmpty { item = self.outputBuffer.removeFirst() } } return item } // ====== 渲染线程相关 ====== /// 启动渲染线程,定时从输出缓冲区取帧并刷新Flutter纹理,支持EMA自适应帧率 private func startRenderThread() { renderThreadRunning = true renderThread = Thread { [weak self] in guard let self = self else { return } while self.renderThreadRunning { let frameIntervalMs = Int(1000.0 / self.smoothedFps) let loopStart = Date().timeIntervalSince1970 * 1000.0 if let (pixelBuffer, timestamp) = self.dequeueOutput() { // 渲染到Flutter纹理 DispatchQueue.main.async { self.onFrameDecoded?(pixelBuffer, timestamp) } // 只在首次渲染时回调Flutter if !self.hasNotifiedFlutter { self.hasNotifiedFlutter = true // 由外部插件层负责onFrameRendered回调 } // 帧率统计 self.renderedTimestamps.append(Int64(Date().timeIntervalSince1970 * 1000)) if self.renderedTimestamps.count > self.renderedTimestampsMaxCount { self.renderedTimestamps.removeFirst() } self.renderedFrameCount += 1 if self.renderedFrameCount % self.fpsAdjustInterval == 0 { let measuredFps = self.calculateDecodeFps() let newFps = self.updateSmoothedFps(measuredFps) self.renderFps = newFps } } // 控制渲染节奏 let loopCost = Int(Date().timeIntervalSince1970 * 1000.0 - loopStart) let sleepMs = frameIntervalMs - loopCost if sleepMs > 0 { Thread.sleep(forTimeInterval: Double(sleepMs) / 1000.0) } } } renderThread?.start() } /// 停止渲染线程 private func stopRenderThread() { renderThreadRunning = false renderThread?.cancel() renderThread = nil } // ====== EMA帧率平滑算法 ====== /// 计算最近N帧的平均解码帧率 private func calculateDecodeFps() -> Double { guard renderedTimestamps.count >= 2 else { return smoothedFps } let first = renderedTimestamps.first! let last = renderedTimestamps.last! let frameCount = renderedTimestamps.count - 1 let durationMs = max(last - first, 1) return Double(frameCount) * 1000.0 / Double(durationMs) } /// EMA平滑更新渲染帧率 private func updateSmoothedFps(_ measuredFps: Double) -> Int { let safeFps = min(max(measuredFps, minFps), maxFps) let targetFps = alpha * safeFps + (1 - alpha) * smoothedFps let delta = targetFps - smoothedFps let step = min(max(delta, -maxStep), maxStep) smoothedFps = min(max(smoothedFps + step, minFps), maxFps) return Int(smoothedFps) } /// 初始化解码会话(首次收到I帧时调用) private func setupSession(sps: Data?, pps: Data?) -> Bool { // 释放旧会话 if let session = decompressionSession { VTDecompressionSessionInvalidate(session) decompressionSession = nil } formatDesc = nil isSessionReady = false guard let sps = sps, let pps = pps else { print("[VideoDecoder] 缺少SPS/PPS,无法初始化解码会话") return false } // 校验SPS/PPS长度和类型 let spsType: UInt8 = sps.count > 0 ? (sps[0] & 0x1F) : 0 let ppsType: UInt8 = pps.count > 0 ? (pps[0] & 0x1F) : 0 if sps.count < 3 || spsType != 7 { print("[VideoDecoder][错误] SPS内容异常,len=\(sps.count), type=\(spsType)") return false } if pps.count < 3 || ppsType != 8 { print("[VideoDecoder][错误] PPS内容异常,len=\(pps.count), type=\(ppsType)") return false } var success = false sps.withUnsafeBytes { spsPtr in pps.withUnsafeBytes { ppsPtr in let parameterSetPointers: [UnsafePointer] = [ spsPtr.baseAddress!.assumingMemoryBound(to: UInt8.self), ppsPtr.baseAddress!.assumingMemoryBound(to: UInt8.self) ] let parameterSetSizes: [Int] = [sps.count, pps.count] let status = CMVideoFormatDescriptionCreateFromH264ParameterSets( allocator: kCFAllocatorDefault, parameterSetCount: 2, parameterSetPointers: parameterSetPointers, parameterSetSizes: parameterSetSizes, nalUnitHeaderLength: 4, formatDescriptionOut: &formatDesc ) if status != noErr { print("[VideoDecoder] 创建FormatDescription失败: \(status)") success = false } else { success = true } } } if !success { return false } var callback = VTDecompressionOutputCallbackRecord( decompressionOutputCallback: { (decompressionOutputRefCon, _, status, _, imageBuffer, pts, _) in let decoder = Unmanaged.fromOpaque(decompressionOutputRefCon!).takeUnretainedValue() if status == noErr, let pixelBuffer = imageBuffer { // 入队到输出缓冲区,由渲染线程拉取 decoder.enqueueOutput((pixelBuffer, Int64(pts.seconds * 1000))) } else { print("[VideoDecoder] 解码回调失败, status=\(status)") } }, decompressionOutputRefCon: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) ) let attrs: [NSString: Any] = [ kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, kCVPixelBufferWidthKey: width, kCVPixelBufferHeightKey: height, kCVPixelBufferOpenGLESCompatibilityKey: true ] let status2 = VTDecompressionSessionCreate( allocator: kCFAllocatorDefault, formatDescription: formatDesc!, decoderSpecification: nil, imageBufferAttributes: attrs as CFDictionary, outputCallback: &callback, decompressionSessionOut: &decompressionSession ) if status2 != noErr { print("[VideoDecoder] 创建解码会话失败: \(status2)") return false } isSessionReady = true print("[VideoDecoder] 解码会话初始化成功") return true } /// 解码一帧数据 func decodeFrame(frameData: Data, frameType: Int, timestamp: Int64, frameSeq: Int, refIFrameSeq: Int?, sps: Data? = nil, pps: Data? = nil) { enqueueInput((frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps)) // 解码线程异步处理 decodeQueue.async { [weak self] in guard let self = self else { return } guard let (frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps) = self.dequeueInput() else { return } if !self.isSessionReady, let sps = sps, let pps = pps { guard self.setupSession(sps: sps, pps: pps) else { return } } guard let session = self.decompressionSession else { return } guard frameData.count > 4 else { return } var avccData = frameData let naluLen = UInt32(frameData.count - 4).bigEndian if avccData.count >= 4 { avccData.replaceSubrange(0..<4, with: withUnsafeBytes(of: naluLen) { Data($0) }) } else { return } var blockBuffer: CMBlockBuffer? let status = CMBlockBufferCreateWithMemoryBlock( allocator: kCFAllocatorDefault, memoryBlock: UnsafeMutableRawPointer(mutating: (avccData as NSData).bytes), blockLength: avccData.count, blockAllocator: kCFAllocatorNull, customBlockSource: nil, offsetToData: 0, dataLength: avccData.count, flags: 0, blockBufferOut: &blockBuffer ) if status != kCMBlockBufferNoErr { return } var sampleBuffer: CMSampleBuffer? var timing = CMSampleTimingInfo(duration: .invalid, presentationTimeStamp: CMTime(value: timestamp, timescale: 1000), decodeTimeStamp: .invalid) let status2 = CMSampleBufferCreate( allocator: kCFAllocatorDefault, dataBuffer: blockBuffer, dataReady: true, makeDataReadyCallback: nil, refcon: nil, formatDescription: self.formatDesc, sampleCount: 1, sampleTimingEntryCount: 1, sampleTimingArray: &timing, sampleSizeEntryCount: 1, sampleSizeArray: [avccData.count], sampleBufferOut: &sampleBuffer ) if status2 != noErr { return } let decodeFlags: VTDecodeFrameFlags = [] var infoFlags = VTDecodeInfoFlags() let status3 = VTDecompressionSessionDecodeFrame( session, sampleBuffer: sampleBuffer!, flags: decodeFlags, frameRefcon: nil, infoFlagsOut: &infoFlags ) if status3 != noErr { print("[VideoDecoder] 解码失败: \(status3)") } } } /// 释放解码器资源 func release() { stopRenderThread() decodeQueue.sync { if let session = decompressionSession { VTDecompressionSessionInvalidate(session) } decompressionSession = nil formatDesc = nil isSessionReady = false frameSeqSet.removeAll() lastIFrameSeq = nil } inputQueue.async(flags: .barrier) { self.inputBuffer.removeAll() } outputQueue.async(flags: .barrier) { self.outputBuffer.removeAll() } print("[VideoDecoder] 解码器已释放") } }