fix:调整ios的解码流程
This commit is contained in:
parent
5dfbd190fd
commit
1370536da9
@ -42,9 +42,11 @@ public class VideoDecodePlugin: NSObject, FlutterPlugin, FlutterTexture {
|
||||
let width = args["width"] as? Int,
|
||||
let height = args["height"] as? Int,
|
||||
let codecType = args["codecType"] as? String else {
|
||||
print("[VideoDecodePlugin][错误] 参数解析失败：\(String(describing: call.arguments))")
|
||||
result(FlutterError(code: "INVALID_ARGS", message: "参数错误", details: nil))
|
||||
return
|
||||
}
|
||||
|
||||
// 释放旧解码器和纹理
|
||||
decoder?.release()
|
||||
decoder = nil
|
||||
@ -52,13 +54,17 @@ public class VideoDecodePlugin: NSObject, FlutterPlugin, FlutterTexture {
|
||||
textureRegistry?.unregisterTexture(tid)
|
||||
textureId = nil
|
||||
}
|
||||
|
||||
// 注册Flutter纹理
|
||||
guard let registry = textureRegistry else {
|
||||
print("[VideoDecodePlugin][错误] 无法获取纹理注册表")
|
||||
result(FlutterError(code: "NO_TEXTURE_REGISTRY", message: "无法获取纹理注册表", details: nil))
|
||||
return
|
||||
}
|
||||
|
||||
let textureId = registry.register(self)
|
||||
self.textureId = textureId
|
||||
|
||||
// 创建解码器
|
||||
let decoder = VideoDecoder(width: width, height: height, codecType: codecType)
|
||||
self.decoder = decoder
|
||||
@ -79,32 +85,94 @@ public class VideoDecodePlugin: NSObject, FlutterPlugin, FlutterTexture {
|
||||
result(textureId)
|
||||
}
|
||||
|
||||
/// 新增：去除NALU起始码的工具方法（增强日志与健壮性）
|
||||
/// 去除NALU起始码的工具方法
|
||||
private func stripStartCode(_ data: Data) -> Data {
|
||||
let originalLen = data.count
|
||||
let naluType: UInt8 = {
|
||||
if data.count > 4 && data[0] == 0x00 && data[1] == 0x00 && data[2] == 0x00 && data[3] == 0x01 {
|
||||
return data[4] & 0x1F
|
||||
} else if data.count > 3 && data[0] == 0x00 && data[1] == 0x00 && data[2] == 0x01 {
|
||||
return data[3] & 0x1F
|
||||
}
|
||||
return 0
|
||||
}()
|
||||
var stripped: Data = data
|
||||
if data.count > 4 && data[0] == 0x00 && data[1] == 0x00 && data[2] == 0x00 && data[3] == 0x01 {
|
||||
stripped = data.subdata(in: 4..<data.count)
|
||||
} else if data.count > 3 && data[0] == 0x00 && data[1] == 0x00 && data[2] == 0x01 {
|
||||
stripped = data.subdata(in: 3..<data.count)
|
||||
}
|
||||
let strippedLen = stripped.count
|
||||
let strippedType: UInt8 = stripped.count > 0 ? (stripped[0] & 0x1F) : 0
|
||||
if strippedLen < 3 || (strippedType != 7 && strippedType != 8) {
|
||||
print("[VideoDecodePlugin][警告] strip后NALU长度或类型异常，type=", strippedType, "len=", strippedLen)
|
||||
}
|
||||
// 只在异常时输出警告，不再输出详细内容
|
||||
return stripped
|
||||
}
|
||||
|
||||
// 修改：更合理的side_data检测工具
|
||||
private func checkNaluForSideData(_ nalu: Data, naluType: UInt8) -> Bool {
|
||||
let maxSize = naluType == 5 ? 150_000 : 30_000 // I帧最大150KB，P帧最大30KB
|
||||
if nalu.count > maxSize {
|
||||
print("[VideoDecodePlugin][警告] NALU长度异常，可能包含side_data，type=\(naluType)，len=\(nalu.count)")
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// 简化：提取第一个合法NALU工具
|
||||
private func extractFirstValidNalu(_ nalu: Data) -> Data {
|
||||
// 查找第一个起始码（支持3字节和4字节格式）
|
||||
var start = -1
|
||||
var startCodeLength = 0
|
||||
|
||||
// 检查4字节起始码
|
||||
if nalu.count >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00 && nalu[2] == 0x00 && nalu[3] == 0x01 {
|
||||
start = 0
|
||||
startCodeLength = 4
|
||||
}
|
||||
// 检查3字节起始码
|
||||
else if nalu.count >= 3 && nalu[0] == 0x00 && nalu[1] == 0x00 && nalu[2] == 0x01 {
|
||||
start = 0
|
||||
startCodeLength = 3
|
||||
}
|
||||
// 在数据中间查找起始码
|
||||
else {
|
||||
for i in 0..<(nalu.count - 4) {
|
||||
if nalu[i] == 0x00 && nalu[i + 1] == 0x00 {
|
||||
if nalu[i + 2] == 0x00 && nalu[i + 3] == 0x01 {
|
||||
start = i
|
||||
startCodeLength = 4
|
||||
break
|
||||
} else if nalu[i + 2] == 0x01 {
|
||||
start = i
|
||||
startCodeLength = 3
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if start == -1 {
|
||||
print("[VideoDecodePlugin][警告] NALU无AnnexB起始码，丢弃该帧")
|
||||
return Data()
|
||||
}
|
||||
|
||||
let searchRange = (start + startCodeLength)..<nalu.count
|
||||
var end = nalu.count
|
||||
|
||||
// 查找下一个起始码（同时支持3字节和4字节格式）
|
||||
for i in searchRange.lowerBound..<(nalu.count - 3) {
|
||||
if nalu[i] == 0x00 && nalu[i + 1] == 0x00 {
|
||||
if i + 3 < nalu.count && nalu[i + 2] == 0x00 && nalu[i + 3] == 0x01 {
|
||||
end = i
|
||||
break
|
||||
} else if nalu[i + 2] == 0x01 {
|
||||
end = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let extractedNalu = nalu[start..<end]
|
||||
|
||||
// 验证提取的NALU
|
||||
if extractedNalu.count >= startCodeLength + 1 {
|
||||
let naluType = extractedNalu[startCodeLength] & 0x1F
|
||||
if checkNaluForSideData(extractedNalu, naluType: naluType) {
|
||||
return Data()
|
||||
}
|
||||
}
|
||||
|
||||
return extractedNalu
|
||||
}
|
||||
|
||||
/// 解码视频帧
|
||||
private func handleDecodeFrame(call: FlutterMethodCall, result: @escaping FlutterResult) {
|
||||
guard let args = call.arguments as? [String: Any],
|
||||
@ -128,30 +196,32 @@ public class VideoDecodePlugin: NSObject, FlutterPlugin, FlutterTexture {
|
||||
return 0
|
||||
}()
|
||||
|
||||
print("[VideoDecodePlugin][调试] handleDecodeFrame: frameType=\(frameType), naluType=\(naluType), cachedSpsLen=\(cachedSps?.count ?? 0), cachedPpsLen=\(cachedPps?.count ?? 0)")
|
||||
|
||||
// 缓存SPS/PPS（去除起始码）并立即尝试初始化解码器
|
||||
// 缓存SPS/PPS（去除起始码）
|
||||
if naluType == 7 { // SPS
|
||||
// 只缓存，不再输出详细日志
|
||||
cachedSps = stripStartCode(data)
|
||||
result(true)
|
||||
return
|
||||
} else if naluType == 8 { // PPS
|
||||
// 只缓存，不再输出详细日志
|
||||
cachedPps = stripStartCode(data)
|
||||
result(true)
|
||||
return
|
||||
} else if naluType == 5 { // IDR/I帧
|
||||
// 先提取第一个合法NALU，直接推送
|
||||
// 提取第一个合法NALU
|
||||
let firstNalu = extractFirstValidNalu(data)
|
||||
if firstNalu.isEmpty { return }
|
||||
print("[VideoDecodePlugin] 发送I帧, 长度: \(firstNalu.count), 头部: \(firstNalu.prefix(8).map { String(format: "%02X", $0) }.joined(separator: " ")), cachedSps长度: \(cachedSps?.count ?? 0), cachedPps长度: \(cachedPps?.count ?? 0)")
|
||||
if firstNalu.isEmpty {
|
||||
result(false)
|
||||
return
|
||||
}
|
||||
|
||||
decoder?.decodeFrame(frameData: firstNalu, frameType: frameType, timestamp: Int64(timestamp), frameSeq: frameSeq, refIFrameSeq: refIFrameSeq, sps: cachedSps, pps: cachedPps)
|
||||
} else {
|
||||
// 先提取第一个合法NALU，直接推送
|
||||
// 提取第一个合法NALU
|
||||
let firstNalu = extractFirstValidNalu(data)
|
||||
if firstNalu.isEmpty { return }
|
||||
print("[VideoDecodePlugin] 发送P/B帧, 长度: \(firstNalu.count), 头部: \(firstNalu.prefix(8).map { String(format: "%02X", $0) }.joined(separator: " "))")
|
||||
if firstNalu.isEmpty {
|
||||
result(false)
|
||||
return
|
||||
}
|
||||
|
||||
decoder?.decodeFrame(frameData: firstNalu, frameType: frameType, timestamp: Int64(timestamp), frameSeq: frameSeq, refIFrameSeq: refIFrameSeq)
|
||||
}
|
||||
result(true)
|
||||
@ -182,28 +252,4 @@ public class VideoDecodePlugin: NSObject, FlutterPlugin, FlutterTexture {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// 新增side_data检测工具
|
||||
private func checkNaluForSideData(_ nalu: Data, naluType: UInt8) -> Bool {
|
||||
if (naluType == 5 && nalu.count > 10000) || (naluType != 7 && naluType != 8 && nalu.count > 10000) {
|
||||
print("[VideoDecodePlugin][警告] NALU长度异常，可能包含side_data，type=\(naluType)，len=\(nalu.count)")
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// 新增：提取第一个合法NALU工具
|
||||
private func extractFirstValidNalu(_ nalu: Data) -> Data {
|
||||
guard let start = nalu.range(of: Data([0x00, 0x00, 0x00, 0x01]))?.lowerBound else {
|
||||
print("[VideoDecodePlugin][警告] NALU无AnnexB起始码，丢弃该帧")
|
||||
return Data()
|
||||
}
|
||||
let searchRange = (start+4)..<nalu.count
|
||||
if let next = nalu[searchRange].range(of: Data([0x00, 0x00, 0x00, 0x01]))?.lowerBound {
|
||||
let end = searchRange.lowerBound + next
|
||||
return nalu[start..<end]
|
||||
} else {
|
||||
return nalu[start..<nalu.count]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -35,49 +35,49 @@ class VideoDecoder {
|
||||
/// 已处理帧序号集合
|
||||
private var frameSeqSet = Set<Int>()
|
||||
/// 最大允许延迟（毫秒）
|
||||
private let maxAllowedDelayMs: Int = 350
|
||||
private let maxAllowedDelayMs: Int64 = 750 // 与Android端保持一致
|
||||
/// 时间戳基准
|
||||
private var timestampBaseMs: Int64?
|
||||
/// 首帧相对时间戳
|
||||
private var firstFrameRelativeTimestamp: Int64?
|
||||
|
||||
// ====== 新增：缓冲区与自适应帧率相关成员 ======
|
||||
// ====== 缓冲区与渲染相关成员 ======
|
||||
/// 输入缓冲区（待解码帧队列），线程安全
|
||||
private let inputQueue = DispatchQueue(label: "video_decode_plugin.input.queue", attributes: .concurrent)
|
||||
private var inputBuffer: [(frameData: Data, frameType: Int, timestamp: Int64, frameSeq: Int, refIFrameSeq: Int?, sps: Data?, pps: Data?)] = []
|
||||
private let inputBufferSemaphore = DispatchSemaphore(value: 1)
|
||||
private let inputBufferMaxCount = 15
|
||||
private let inputBufferMaxCount = 100 // 与Android端保持一致
|
||||
/// 输出缓冲区（解码后帧队列），线程安全
|
||||
private let outputQueue = DispatchQueue(label: "video_decode_plugin.output.queue", attributes: .concurrent)
|
||||
private var outputBuffer: [(pixelBuffer: CVPixelBuffer, timestamp: Int64)] = []
|
||||
private let outputBufferSemaphore = DispatchSemaphore(value: 1)
|
||||
private let outputBufferMaxCount = 15
|
||||
/// 渲染线程
|
||||
private var renderThread: Thread?
|
||||
private let outputBufferMaxCount = 100 // 与Android端保持一致
|
||||
/// 渲染定时器
|
||||
private var renderTimer: DispatchSourceTimer?
|
||||
/// 渲染线程运行标志
|
||||
private var renderThreadRunning = false
|
||||
/// 首次渲染回调标志
|
||||
private var hasNotifiedFlutter = false
|
||||
/// 当前渲染帧率
|
||||
private var renderFps: Int = 15
|
||||
/// EMA平滑后的帧率
|
||||
private var smoothedFps: Double = 15.0
|
||||
/// EMA平滑系数
|
||||
private let alpha: Double = 0.2
|
||||
/// 最小帧率
|
||||
private let minFps: Double = 8.0
|
||||
/// 最大帧率
|
||||
private let maxFps: Double = 30.0
|
||||
/// 单次最大调整幅度
|
||||
private let maxStep: Double = 2.0
|
||||
/// 渲染帧时间戳队列
|
||||
private var renderedTimestamps: [Int64] = [] // ms
|
||||
/// 渲染帧时间戳最大数量
|
||||
private let renderedTimestampsMaxCount = 20
|
||||
/// 已渲染帧计数
|
||||
private var renderedFrameCount = 0
|
||||
/// 每N帧调整一次帧率
|
||||
private let fpsAdjustInterval = 10
|
||||
private var renderFps: Int = 20 // 与Android端保持一致
|
||||
/// 渲染间隔（毫秒）
|
||||
private var renderIntervalMs: Int64 = 0
|
||||
/// 允许的渲染时间抖动范围（毫秒）
|
||||
private let renderJitterMs: Int64 = 2
|
||||
/// 上次渲染时间
|
||||
private var lastRenderTimeMs: Int64 = 0
|
||||
/// 渲染启动标志（低水位控制）
|
||||
private var renderStarted = false
|
||||
|
||||
// ====== 新增：帧重排序与丢帧相关成员 ======
|
||||
/// 帧重排序缓冲区
|
||||
private var reorderBuffer: [Int: (frameData: Data, frameType: Int, timestamp: Int64, frameSeq: Int, refIFrameSeq: Int?, sps: Data?, pps: Data?)] = [:]
|
||||
/// 已收到的I帧序号集合
|
||||
private var receivedIFrames = Set<Int>()
|
||||
/// 重排序缓冲区锁
|
||||
private let reorderLock = NSLock()
|
||||
/// 重排序缓冲区最大容量
|
||||
private let maxReorderBufferSize = 100 // 与输入缓冲区大小一致
|
||||
|
||||
/// 解码回调，输出CVPixelBuffer和时间戳
|
||||
var onFrameDecoded: ((CVPixelBuffer, Int64) -> Void)? = { _, _ in }
|
||||
@ -87,7 +87,9 @@ class VideoDecoder {
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.codecType = CodecType(rawValue: codecType.lowercased()) ?? .h264
|
||||
startRenderThread()
|
||||
self.renderIntervalMs = Int64(1000.0 / Double(renderFps))
|
||||
startRenderTimer()
|
||||
print("[VideoDecoder] 初始化解码器: width=\(width), height=\(height)")
|
||||
}
|
||||
|
||||
// ====== 输入缓冲区操作 ======
|
||||
@ -95,7 +97,8 @@ class VideoDecoder {
|
||||
private func enqueueInput(_ item: (Data, Int, Int64, Int, Int?, Data?, Data?)) {
|
||||
inputQueue.async(flags: .barrier) {
|
||||
if self.inputBuffer.count >= self.inputBufferMaxCount {
|
||||
self.inputBuffer.removeFirst()
|
||||
self.inputBuffer.removeFirst() // 缓冲区满时丢弃最旧帧
|
||||
print("[VideoDecoder][警告] 输入缓冲区满，丢弃最旧帧")
|
||||
}
|
||||
self.inputBuffer.append(item)
|
||||
}
|
||||
@ -115,7 +118,8 @@ class VideoDecoder {
|
||||
private func enqueueOutput(_ item: (CVPixelBuffer, Int64)) {
|
||||
outputQueue.async(flags: .barrier) {
|
||||
if self.outputBuffer.count >= self.outputBufferMaxCount {
|
||||
self.outputBuffer.removeFirst()
|
||||
self.outputBuffer.removeFirst() // 缓冲区满时丢弃最旧帧
|
||||
print("[VideoDecoder][警告] 输出缓冲区满，丢弃最旧帧")
|
||||
}
|
||||
self.outputBuffer.append(item)
|
||||
}
|
||||
@ -130,72 +134,141 @@ class VideoDecoder {
|
||||
}
|
||||
return item
|
||||
}
|
||||
// ====== 渲染线程相关 ======
|
||||
/// 启动渲染线程，定时从输出缓冲区取帧并刷新Flutter纹理，支持EMA自适应帧率
|
||||
private func startRenderThread() {
|
||||
renderThreadRunning = true
|
||||
renderThread = Thread { [weak self] in
|
||||
guard let self = self else { return }
|
||||
while self.renderThreadRunning {
|
||||
let frameIntervalMs = Int(1000.0 / self.smoothedFps)
|
||||
let loopStart = Date().timeIntervalSince1970 * 1000.0
|
||||
if let (pixelBuffer, timestamp) = self.dequeueOutput() {
|
||||
// 渲染到Flutter纹理
|
||||
DispatchQueue.main.async {
|
||||
self.onFrameDecoded?(pixelBuffer, timestamp)
|
||||
}
|
||||
// 只在首次渲染时回调Flutter
|
||||
if !self.hasNotifiedFlutter {
|
||||
self.hasNotifiedFlutter = true
|
||||
// 由外部插件层负责onFrameRendered回调
|
||||
}
|
||||
// 帧率统计
|
||||
self.renderedTimestamps.append(Int64(Date().timeIntervalSince1970 * 1000))
|
||||
if self.renderedTimestamps.count > self.renderedTimestampsMaxCount {
|
||||
self.renderedTimestamps.removeFirst()
|
||||
}
|
||||
self.renderedFrameCount += 1
|
||||
if self.renderedFrameCount % self.fpsAdjustInterval == 0 {
|
||||
let measuredFps = self.calculateDecodeFps()
|
||||
let newFps = self.updateSmoothedFps(measuredFps)
|
||||
self.renderFps = newFps
|
||||
}
|
||||
|
||||
// ====== 帧重排序与依赖管理 ======
|
||||
/// 处理帧重排序和依赖关系
|
||||
private func handleFrameReordering(frameData: Data, frameType: Int, timestamp: Int64, frameSeq: Int, refIFrameSeq: Int?, sps: Data?, pps: Data?) -> Bool {
|
||||
reorderLock.lock()
|
||||
defer { reorderLock.unlock() }
|
||||
|
||||
// 1. 延迟丢帧检查
|
||||
let now = Int64(Date().timeIntervalSince1970 * 1000)
|
||||
let base = timestampBaseMs ?? 0
|
||||
let firstRel = firstFrameRelativeTimestamp ?? 0
|
||||
let absTimestamp = base + (timestamp - firstRel)
|
||||
|
||||
if absTimestamp < now - maxAllowedDelayMs {
|
||||
print("[VideoDecoder][警告] 丢弃延迟帧: type=\(frameType), seq=\(frameSeq), delay=\(now - absTimestamp)ms")
|
||||
return false
|
||||
}
|
||||
|
||||
// 2. 帧重排序处理
|
||||
if frameType == 0 { // I帧
|
||||
receivedIFrames.insert(frameSeq)
|
||||
lastIFrameSeq = frameSeq
|
||||
|
||||
// I帧直接解码
|
||||
enqueueInput((frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps))
|
||||
|
||||
// 检查并解码所有依赖该I帧的P帧
|
||||
let readyPFrames = reorderBuffer.values
|
||||
.filter { $0.refIFrameSeq == frameSeq }
|
||||
.sorted { $0.frameSeq < $1.frameSeq }
|
||||
|
||||
for pFrame in readyPFrames {
|
||||
enqueueInput(pFrame)
|
||||
reorderBuffer.removeValue(forKey: pFrame.frameSeq)
|
||||
}
|
||||
|
||||
if !readyPFrames.isEmpty {
|
||||
print("[VideoDecoder] I帧\(frameSeq)释放\(readyPFrames.count)个P帧")
|
||||
}
|
||||
|
||||
// 清理过期缓存
|
||||
if reorderBuffer.count > maxReorderBufferSize {
|
||||
let toRemove = reorderBuffer.keys.sorted().prefix(reorderBuffer.count - maxReorderBufferSize)
|
||||
for seq in toRemove {
|
||||
reorderBuffer.removeValue(forKey: seq)
|
||||
}
|
||||
// 控制渲染节奏
|
||||
let loopCost = Int(Date().timeIntervalSince1970 * 1000.0 - loopStart)
|
||||
let sleepMs = frameIntervalMs - loopCost
|
||||
if sleepMs > 0 {
|
||||
Thread.sleep(forTimeInterval: Double(sleepMs) / 1000.0)
|
||||
print("[VideoDecoder][警告] 重排序缓冲区溢出，清理\(toRemove.count)个P帧")
|
||||
}
|
||||
|
||||
return true
|
||||
|
||||
} else { // P帧
|
||||
// 检查P帧依赖
|
||||
if let refIFrameSeq = refIFrameSeq, receivedIFrames.contains(refIFrameSeq) {
|
||||
// 依赖的I帧已收到，直接解码
|
||||
enqueueInput((frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps))
|
||||
return true
|
||||
} else {
|
||||
// 依赖的I帧未到，缓存P帧
|
||||
reorderBuffer[frameSeq] = (frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps)
|
||||
print("[VideoDecoder] P帧\(frameSeq)缓存，等待I帧\(refIFrameSeq ?? -1)")
|
||||
|
||||
// 控制缓冲区大小
|
||||
if reorderBuffer.count > maxReorderBufferSize {
|
||||
let toRemove = reorderBuffer.keys.sorted().prefix(reorderBuffer.count - maxReorderBufferSize)
|
||||
for seq in toRemove {
|
||||
reorderBuffer.removeValue(forKey: seq)
|
||||
}
|
||||
print("[VideoDecoder][警告] 重排序缓冲区溢出，清理\(toRemove.count)个P帧")
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ====== 渲染定时器相关 ======
|
||||
/// 启动渲染定时器，定时从输出缓冲区取帧并刷新Flutter纹理
|
||||
private func startRenderTimer() {
|
||||
renderThreadRunning = true
|
||||
let timer = DispatchSource.makeTimerSource(queue: DispatchQueue.global())
|
||||
timer.schedule(deadline: .now(), repeating: .milliseconds(Int(renderIntervalMs / 2)))
|
||||
timer.setEventHandler { [weak self] in
|
||||
guard let self = self else { return }
|
||||
|
||||
let now = Int64(Date().timeIntervalSince1970 * 1000)
|
||||
// 控制渲染间隔，避免过快渲染
|
||||
let timeSinceLastRender = now - self.lastRenderTimeMs
|
||||
if timeSinceLastRender < self.renderIntervalMs - self.renderJitterMs {
|
||||
return
|
||||
}
|
||||
|
||||
// 低水位启动渲染逻辑
|
||||
if !self.renderStarted {
|
||||
var outputCount = 0
|
||||
self.outputQueue.sync { outputCount = self.outputBuffer.count }
|
||||
if outputCount >= Int(Double(self.outputBufferMaxCount) * 0.15) {
|
||||
self.renderStarted = true
|
||||
print("[VideoDecoder] 渲染启动，outputBuffer已达低水位: \(outputCount)")
|
||||
} else {
|
||||
// 未达到低水位前不渲染
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if let (pixelBuffer, timestamp) = self.dequeueOutput() {
|
||||
// 延迟丢帧检查
|
||||
let now = Int64(Date().timeIntervalSince1970 * 1000)
|
||||
let base = timestampBaseMs ?? 0
|
||||
let firstRel = firstFrameRelativeTimestamp ?? 0
|
||||
let absTimestamp = base + (timestamp - firstRel)
|
||||
|
||||
if absTimestamp < now - self.maxAllowedDelayMs {
|
||||
print("[VideoDecoder][警告] 丢弃延迟渲染帧: delay=\(now - absTimestamp)ms")
|
||||
return
|
||||
}
|
||||
|
||||
DispatchQueue.main.async {
|
||||
self.onFrameDecoded?(pixelBuffer, timestamp)
|
||||
}
|
||||
self.lastRenderTimeMs = now
|
||||
if !self.hasNotifiedFlutter {
|
||||
self.hasNotifiedFlutter = true
|
||||
}
|
||||
}
|
||||
}
|
||||
renderThread?.start()
|
||||
timer.resume()
|
||||
renderTimer = timer
|
||||
}
|
||||
/// 停止渲染线程
|
||||
private func stopRenderThread() {
|
||||
/// 停止渲染定时器
|
||||
private func stopRenderTimer() {
|
||||
renderThreadRunning = false
|
||||
renderThread?.cancel()
|
||||
renderThread = nil
|
||||
}
|
||||
// ====== EMA帧率平滑算法 ======
|
||||
/// 计算最近N帧的平均解码帧率
|
||||
private func calculateDecodeFps() -> Double {
|
||||
guard renderedTimestamps.count >= 2 else { return smoothedFps }
|
||||
let first = renderedTimestamps.first!
|
||||
let last = renderedTimestamps.last!
|
||||
let frameCount = renderedTimestamps.count - 1
|
||||
let durationMs = max(last - first, 1)
|
||||
return Double(frameCount) * 1000.0 / Double(durationMs)
|
||||
}
|
||||
/// EMA平滑更新渲染帧率
|
||||
private func updateSmoothedFps(_ measuredFps: Double) -> Int {
|
||||
let safeFps = min(max(measuredFps, minFps), maxFps)
|
||||
let targetFps = alpha * safeFps + (1 - alpha) * smoothedFps
|
||||
let delta = targetFps - smoothedFps
|
||||
let step = min(max(delta, -maxStep), maxStep)
|
||||
smoothedFps = min(max(smoothedFps + step, minFps), maxFps)
|
||||
return Int(smoothedFps)
|
||||
renderTimer?.cancel()
|
||||
renderTimer = nil
|
||||
}
|
||||
|
||||
/// 初始化解码会话（首次收到I帧时调用）
|
||||
private func setupSession(sps: Data?, pps: Data?) -> Bool {
|
||||
// 释放旧会话
|
||||
@ -206,9 +279,10 @@ class VideoDecoder {
|
||||
formatDesc = nil
|
||||
isSessionReady = false
|
||||
guard let sps = sps, let pps = pps else {
|
||||
print("[VideoDecoder] 缺少SPS/PPS，无法初始化解码会话")
|
||||
print("[VideoDecoder][错误] 缺少SPS/PPS，无法初始化解码会话")
|
||||
return false
|
||||
}
|
||||
|
||||
// 校验SPS/PPS长度和类型
|
||||
let spsType: UInt8 = sps.count > 0 ? (sps[0] & 0x1F) : 0
|
||||
let ppsType: UInt8 = pps.count > 0 ? (pps[0] & 0x1F) : 0
|
||||
@ -220,6 +294,7 @@ class VideoDecoder {
|
||||
print("[VideoDecoder][错误] PPS内容异常，len=\(pps.count), type=\(ppsType)")
|
||||
return false
|
||||
}
|
||||
|
||||
var success = false
|
||||
sps.withUnsafeBytes { spsPtr in
|
||||
pps.withUnsafeBytes { ppsPtr in
|
||||
@ -237,7 +312,7 @@ class VideoDecoder {
|
||||
formatDescriptionOut: &formatDesc
|
||||
)
|
||||
if status != noErr {
|
||||
print("[VideoDecoder] 创建FormatDescription失败: \(status)")
|
||||
print("[VideoDecoder][错误] 创建FormatDescription失败: \(status)")
|
||||
success = false
|
||||
} else {
|
||||
success = true
|
||||
@ -252,7 +327,7 @@ class VideoDecoder {
|
||||
// 入队到输出缓冲区，由渲染线程拉取
|
||||
decoder.enqueueOutput((pixelBuffer, Int64(pts.seconds * 1000)))
|
||||
} else {
|
||||
print("[VideoDecoder] 解码回调失败, status=\(status)")
|
||||
print("[VideoDecoder][错误] 解码回调失败: \(status)")
|
||||
}
|
||||
},
|
||||
decompressionOutputRefCon: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())
|
||||
@ -261,8 +336,10 @@ class VideoDecoder {
|
||||
kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
|
||||
kCVPixelBufferWidthKey: width,
|
||||
kCVPixelBufferHeightKey: height,
|
||||
kCVPixelBufferOpenGLESCompatibilityKey: true
|
||||
kCVPixelBufferOpenGLESCompatibilityKey: true,
|
||||
kCVPixelBufferIOSurfacePropertiesKey: [:]
|
||||
]
|
||||
|
||||
let status2 = VTDecompressionSessionCreate(
|
||||
allocator: kCFAllocatorDefault,
|
||||
formatDescription: formatDesc!,
|
||||
@ -272,7 +349,7 @@ class VideoDecoder {
|
||||
decompressionSessionOut: &decompressionSession
|
||||
)
|
||||
if status2 != noErr {
|
||||
print("[VideoDecoder] 创建解码会话失败: \(status2)")
|
||||
print("[VideoDecoder][错误] 创建解码会话失败: \(status2)")
|
||||
return false
|
||||
}
|
||||
isSessionReady = true
|
||||
@ -282,38 +359,91 @@ class VideoDecoder {
|
||||
|
||||
/// 解码一帧数据
|
||||
func decodeFrame(frameData: Data, frameType: Int, timestamp: Int64, frameSeq: Int, refIFrameSeq: Int?, sps: Data? = nil, pps: Data? = nil) {
|
||||
enqueueInput((frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps))
|
||||
// 解码线程异步处理
|
||||
// 1. 初始化时间戳基准
|
||||
if timestampBaseMs == nil {
|
||||
timestampBaseMs = Int64(Date().timeIntervalSince1970 * 1000)
|
||||
firstFrameRelativeTimestamp = timestamp
|
||||
print("[VideoDecoder] 设置时间戳基准: base=\(timestampBaseMs!), firstRel=\(firstFrameRelativeTimestamp!)")
|
||||
}
|
||||
|
||||
// 2. 帧重排序和依赖管理
|
||||
if !handleFrameReordering(frameData: frameData, frameType: frameType, timestamp: timestamp, frameSeq: frameSeq, refIFrameSeq: refIFrameSeq, sps: sps, pps: pps) {
|
||||
return
|
||||
}
|
||||
|
||||
// 3. 解码处理（由inputQueue触发）
|
||||
decodeQueue.async { [weak self] in
|
||||
guard let self = self else { return }
|
||||
guard let (frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps) = self.dequeueInput() else { return }
|
||||
|
||||
if !self.isSessionReady, let sps = sps, let pps = pps {
|
||||
guard self.setupSession(sps: sps, pps: pps) else { return }
|
||||
}
|
||||
guard let session = self.decompressionSession else { return }
|
||||
guard frameData.count > 4 else { return }
|
||||
var avccData = frameData
|
||||
let naluLen = UInt32(frameData.count - 4).bigEndian
|
||||
if avccData.count >= 4 {
|
||||
avccData.replaceSubrange(0..<4, with: withUnsafeBytes(of: naluLen) { Data($0) })
|
||||
guard frameData.count > 3 else { return }
|
||||
|
||||
// 检查并移除AnnexB起始码
|
||||
var startCodeSize = 0
|
||||
var naluData: Data
|
||||
|
||||
if frameData.count >= 4 && frameData[0] == 0x00 && frameData[1] == 0x00 && frameData[2] == 0x00 && frameData[3] == 0x01 {
|
||||
startCodeSize = 4
|
||||
naluData = frameData.subdata(in: 4..<frameData.count)
|
||||
} else if frameData.count >= 3 && frameData[0] == 0x00 && frameData[1] == 0x00 && frameData[2] == 0x01 {
|
||||
startCodeSize = 3
|
||||
naluData = frameData.subdata(in: 3..<frameData.count)
|
||||
} else {
|
||||
return
|
||||
print("[VideoDecoder][警告] 未找到起始码")
|
||||
naluData = frameData
|
||||
}
|
||||
|
||||
// 创建AVCC格式数据
|
||||
let naluLength = UInt32(naluData.count).bigEndian
|
||||
var avccData = Data(capacity: naluData.count + 4)
|
||||
withUnsafeBytes(of: naluLength) { ptr in
|
||||
avccData.append(ptr.baseAddress!.assumingMemoryBound(to: UInt8.self), count: 4)
|
||||
}
|
||||
avccData.append(naluData)
|
||||
|
||||
var blockBuffer: CMBlockBuffer?
|
||||
let status = CMBlockBufferCreateWithMemoryBlock(
|
||||
allocator: kCFAllocatorDefault,
|
||||
memoryBlock: UnsafeMutableRawPointer(mutating: (avccData as NSData).bytes),
|
||||
memoryBlock: nil,
|
||||
blockLength: avccData.count,
|
||||
blockAllocator: kCFAllocatorNull,
|
||||
blockAllocator: nil,
|
||||
customBlockSource: nil,
|
||||
offsetToData: 0,
|
||||
dataLength: avccData.count,
|
||||
flags: 0,
|
||||
flags: kCMBlockBufferAssureMemoryNowFlag,
|
||||
blockBufferOut: &blockBuffer
|
||||
)
|
||||
if status != kCMBlockBufferNoErr { return }
|
||||
if status != kCMBlockBufferNoErr {
|
||||
print("[VideoDecoder][错误] 创建BlockBuffer失败: \(status)")
|
||||
return
|
||||
}
|
||||
|
||||
// 复制数据到BlockBuffer
|
||||
if let blockBuffer = blockBuffer {
|
||||
let status2 = avccData.withUnsafeBytes { ptr in
|
||||
CMBlockBufferReplaceDataBytes(
|
||||
with: ptr.baseAddress!,
|
||||
blockBuffer: blockBuffer,
|
||||
offsetIntoDestination: 0,
|
||||
dataLength: avccData.count
|
||||
)
|
||||
}
|
||||
if status2 != kCMBlockBufferNoErr {
|
||||
print("[VideoDecoder][错误] 复制数据到BlockBuffer失败: \(status2)")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var sampleBuffer: CMSampleBuffer?
|
||||
var timing = CMSampleTimingInfo(duration: .invalid, presentationTimeStamp: CMTime(value: timestamp, timescale: 1000), decodeTimeStamp: .invalid)
|
||||
var timing = CMSampleTimingInfo(
|
||||
duration: .invalid,
|
||||
presentationTimeStamp: CMTime(value: timestamp, timescale: 1000),
|
||||
decodeTimeStamp: .invalid
|
||||
)
|
||||
let status2 = CMSampleBufferCreate(
|
||||
allocator: kCFAllocatorDefault,
|
||||
dataBuffer: blockBuffer,
|
||||
@ -328,8 +458,12 @@ class VideoDecoder {
|
||||
sampleSizeArray: [avccData.count],
|
||||
sampleBufferOut: &sampleBuffer
|
||||
)
|
||||
if status2 != noErr { return }
|
||||
let decodeFlags: VTDecodeFrameFlags = []
|
||||
if status2 != noErr {
|
||||
print("[VideoDecoder][错误] 创建SampleBuffer失败: \(status2)")
|
||||
return
|
||||
}
|
||||
|
||||
let decodeFlags: VTDecodeFrameFlags = [._EnableAsynchronousDecompression]
|
||||
var infoFlags = VTDecodeInfoFlags()
|
||||
let status3 = VTDecompressionSessionDecodeFrame(
|
||||
session,
|
||||
@ -339,14 +473,19 @@ class VideoDecoder {
|
||||
infoFlagsOut: &infoFlags
|
||||
)
|
||||
if status3 != noErr {
|
||||
print("[VideoDecoder] 解码失败: \(status3)")
|
||||
print("[VideoDecoder][错误] 解码失败: \(status3)")
|
||||
if status3 == -6661 {
|
||||
print(" - 错误类型: kVTInvalidSessionErr (解码会话无效)")
|
||||
print(" - 会话状态: \(self.isSessionReady ? "就绪" : "未就绪")")
|
||||
print(" - formatDesc: \(self.formatDesc != nil ? "有效" : "无效")")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 释放解码器资源
|
||||
func release() {
|
||||
stopRenderThread()
|
||||
stopRenderTimer()
|
||||
decodeQueue.sync {
|
||||
if let session = decompressionSession {
|
||||
VTDecompressionSessionInvalidate(session)
|
||||
@ -356,6 +495,12 @@ class VideoDecoder {
|
||||
isSessionReady = false
|
||||
frameSeqSet.removeAll()
|
||||
lastIFrameSeq = nil
|
||||
|
||||
// 清理重排序相关资源
|
||||
reorderLock.lock()
|
||||
reorderBuffer.removeAll()
|
||||
receivedIFrames.removeAll()
|
||||
reorderLock.unlock()
|
||||
}
|
||||
inputQueue.async(flags: .barrier) { self.inputBuffer.removeAll() }
|
||||
outputQueue.async(flags: .barrier) { self.outputBuffer.removeAll() }
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user