Compare commits

...

10 Commits

4 changed files with 490 additions and 231 deletions

View File

@ -49,7 +49,7 @@ class VideoDecoder(
companion object {
private const val TAG = "VideoDecoder"
private const val TIMEOUT_US = 10000L
private const val INPUT_BUFFER_QUEUE_CAPACITY = 50 // 输入缓冲区容量
private const val BUFFER_QUEUE_CAPACITY = 50 // 增大输入缓冲区容量
}
// region 成员变量定义
@ -60,21 +60,25 @@ class VideoDecoder(
private var mediaCodec: MediaCodec? = null
// 输入帧队列,支持并发,容量较大以防止丢帧
private val inputFrameQueue = LinkedBlockingQueue<FrameData>(INPUT_BUFFER_QUEUE_CAPACITY)
private val inputFrameQueue = LinkedBlockingQueue<FrameData>(BUFFER_QUEUE_CAPACITY)
private var running = true // 解码器运行状态
private val frameSeqSet = Collections.newSetFromMap(ConcurrentHashMap<Int, Boolean>()) // 防止重复帧入队
// 解码输出缓冲区,容量为100帧
private val outputFrameQueue = LinkedBlockingQueue<DecodedFrame>(50)
// 解码输出缓冲区,增大容量
private val outputFrameQueue = LinkedBlockingQueue<DecodedFrame>(BUFFER_QUEUE_CAPACITY)
// 渲染线程控制
// 定时渲染调度器
private var scheduler = Executors.newSingleThreadScheduledExecutor()
private var lastRenderTimeMs = 0L // 记录上次渲染时间
// 这些变量移到init块中因为它们依赖renderFps
private var renderIntervalMs: Long = 0
private val renderJitterMs = 2L // 允许的渲染时间抖动范围
// 主线程Handler用于安全切换onFrameRendered到主线程
private val mainHandler = Handler(Looper.getMainLooper())
// 渲染帧率fps可由外部控制默认18
// 渲染帧率fps可由外部控制默认30
@Volatile
var renderFps: Int = 20
@ -82,22 +86,20 @@ class VideoDecoder(
@Volatile
private var lastIFrameSeq: Int? = null
// 解码输出帧时间戳队列(用于动态帧率统计和平滑)
private val decodeTimestampQueue = ArrayDeque<Long>(20) // 最多保存20帧时间戳
private val decodeTimestampLock = ReentrantLock() // 线程安全保护
// EMA平滑参数
@Volatile
private var smoothedFps: Double = 25.0 // 平滑后的渲染帧率
private val alpha = 0.2 // EMA平滑系数越大响应越快
private val minFps = 8 // 渲染帧率下限,防止过低
private val maxFps = 30 // 渲染帧率上限,防止过高
private val maxStep = 2.0 // 单次最大调整幅度,防止突变
// ====== 动态帧率统计与平滑相关内容已废弃,以下变量保留注释以便后续扩展 ======
// private val decodeTimestampQueue = ArrayDeque<Long>(20) // 最多保存20帧时间戳
// private val decodeTimestampLock = ReentrantLock() // 线程安全保护
// @Volatile
// private var smoothedFps: Double = 25.0 // 平滑后的渲染帧率
// private val alpha = 0.2 // EMA平滑系数越大响应越快
// private val minFps = 8 // 渲染帧率下限,防止过低
// private val maxFps = 30 // 渲染帧率上限,防止过高
// private val maxStep = 2.0 // 单次最大调整幅度,防止突变
// 1. 新增成员变量
@Volatile
private var latestRenderedTimestampMs: Long? = null
private val MAX_ALLOWED_DELAY_MS = 550 // 最大允许延迟,单位毫秒
private val MAX_ALLOWED_DELAY_MS = 1000 // 最大允许延迟,单位毫秒
@Volatile
private var timestampBaseMs: Long? = null
@Volatile
@ -107,7 +109,11 @@ class VideoDecoder(
private val reorderBuffer = mutableMapOf<Int, FrameData>() // key: frameSeq
private val receivedIFrames = mutableSetOf<Int>() // 已收到的I帧frameSeq
private val reorderLock = ReentrantLock() // 线程安全
private val MAX_REORDER_BUFFER_SIZE = 50
private val MAX_REORDER_BUFFER_SIZE = BUFFER_QUEUE_CAPACITY
// 低水位启动渲染标志
@Volatile
private var renderStarted = false
// 输入帧结构体
private data class FrameData(
@ -130,6 +136,9 @@ class VideoDecoder(
// region 初始化与解码器配置
init {
// 初始化渲染相关参数
renderIntervalMs = (1000.0 / renderFps).toLong()
// 配置Surface尺寸
surfaceTexture.setDefaultBufferSize(width, height)
// 选择MIME类型
@ -141,11 +150,54 @@ class VideoDecoder(
// 创建并配置MediaFormat
val format = MediaFormat.createVideoFormat(mime, width, height)
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, width * height)
format.setInteger(MediaFormat.KEY_FRAME_RATE, renderFps)
format.setInteger(MediaFormat.KEY_LOW_LATENCY, 1);
// 移除可能导致flush的配置
// format.setInteger(MediaFormat.KEY_FRAME_RATE, renderFps)
// format.setInteger(MediaFormat.KEY_OPERATING_RATE, renderFps)
format.setInteger(MediaFormat.KEY_LOW_LATENCY, 1)
format.setInteger(MediaFormat.KEY_PRIORITY, 0)
format.setInteger(MediaFormat.KEY_MAX_B_FRAMES, 0)
format.setInteger(MediaFormat.KEY_FRAME_RATE, renderFps);
// 高通解码器特定配置
format.setInteger("vendor.qti-ext-dec-low-latency.enable", 1)
format.setInteger("vendor.qti-ext-dec-picture-order.enable", 0)
format.setInteger("vendor.qti-ext-dec-timestamp-mode.value", 0) // 使用原始时间戳
format.setInteger("vendor.qti-ext-dec-drv-flush.disable", 1) // 禁用驱动层flush
// 创建解码器
val decoder = MediaCodec.createDecoderByType(mime)
// ========== 输出所有支持的视频解码器及类型 ==========
try {
val codecList = if (android.os.Build.VERSION.SDK_INT >= 21) {
android.media.MediaCodecList(android.media.MediaCodecList.ALL_CODECS).codecInfos
} else {
arrayOf<android.media.MediaCodecInfo>()
}
Log.i(TAG, "[CodecList] 支持的视频解码器如下:")
for (info in codecList) {
if (!info.isEncoder) {
val types = info.supportedTypes.joinToString(", ")
val name = info.name
val isHardware = if (android.os.Build.VERSION.SDK_INT >= 29) info.isHardwareAccelerated else !name.startsWith("OMX.google.")
val isSoftware = if (android.os.Build.VERSION.SDK_INT >= 29) info.isSoftwareOnly else name.startsWith("OMX.google.")
Log.i(TAG, "解码器名称: $name, 支持类型: [$types], 硬解码: $isHardware, 软解码: $isSoftware")
}
}
} catch (e: Exception) {
Log.w(TAG, "[CodecList] 获取解码器列表失败", e)
}
// ========== 输出当前创建的解码器类型 ==========
try {
val codecInfo = decoder.codecInfo
val name = codecInfo.name
val isHardware = if (android.os.Build.VERSION.SDK_INT >= 29) codecInfo.isHardwareAccelerated else !name.startsWith("OMX.google.")
val isSoftware = if (android.os.Build.VERSION.SDK_INT >= 29) codecInfo.isSoftwareOnly else name.startsWith("OMX.google.")
Log.i(TAG, "[CurrentCodec] 当前解码器: $name, 硬解码: $isHardware, 软解码: $isSoftware")
} catch (e: Exception) {
Log.w(TAG, "[CurrentCodec] 获取当前解码器信息失败", e)
}
// 设置解码回调
decoder.setCallback(object : MediaCodec.Callback() {
override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {
@ -159,7 +211,7 @@ class VideoDecoder(
val absTimestamp = base + (frame.timestamp - firstRel)
val now = System.currentTimeMillis()
if (absTimestamp < now - MAX_ALLOWED_DELAY_MS) {
Log.w(TAG, "[onInputBufferAvailable] Drop frame due to delay: absFrameTs=$absTimestamp, now=$now, maxDelay=$MAX_ALLOWED_DELAY_MS")
Log.w(TAG, "[onInputBufferAvailable] 丢弃延迟帧: absFrameTs=$absTimestamp, now=$now, maxDelay=$MAX_ALLOWED_DELAY_MS, seq=${frame.frameSeq}")
frameSeqSet.remove(frame.frameSeq)
codec.queueInputBuffer(index, 0, 0, 0, 0)
return
@ -171,6 +223,7 @@ class VideoDecoder(
inputBuffer.put(frame.data)
val start = System.nanoTime()
val ptsUs = absTimestamp * 1000L // 6. 送入解码器用绝对时间戳
// Log.d(TAG, "[MediaCodec] 入解码: type=${frame.frameType}, seq=${frame.frameSeq}, pts=$ptsUs, bufferIdx=$index")
codec.queueInputBuffer(
index,
0,
@ -187,14 +240,7 @@ class VideoDecoder(
}
override fun onOutputBufferAvailable(codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) {
if (!running) return
// 记录解码输出时间戳
val now = SystemClock.elapsedRealtime()
decodeTimestampLock.withLock {
if (decodeTimestampQueue.size >= 20) {
decodeTimestampQueue.removeFirst()
}
decodeTimestampQueue.addLast(now)
}
// ====== 动态帧率统计相关代码已注释,如需帧率统计可恢复 ======
// 解码后帧入输出缓冲区,由渲染线程处理
val frame = DecodedFrame(codec, index, MediaCodec.BufferInfo().apply {
set(0, info.size, info.presentationTimeUs, info.flags)
@ -203,7 +249,9 @@ class VideoDecoder(
// 缓冲区满,丢弃最旧帧再插入
outputFrameQueue.poll()
outputFrameQueue.offer(frame)
Log.w(TAG, "[MediaCodec] outputFrameQueue溢出丢弃最旧帧当前队列=${outputFrameQueue.size}")
}
// Log.d(TAG, "[MediaCodec] 输出: bufferIdx=$index, pts=${info.presentationTimeUs}, 当前outputFrameQueue=${outputFrameQueue.size}")
}
override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
Log.e(TAG, "MediaCodec error", e)
@ -214,29 +262,51 @@ class VideoDecoder(
decoder.start()
mediaCodec = decoder
// 启动定时渲染任务,实现完全线性调度
// 说明本方案通过ScheduledExecutorService定时驱动渲染每帧间隔严格等距不依赖阻塞或sleep
// 优化渲染任务调度
var hasNotifiedFlutter = false
var renderedFrameCount = 0 // 渲染帧计数器
var renderedFrameCount = 0
val renderTask = Runnable {
try {
val now = System.currentTimeMillis()
// 控制渲染间隔,避免过快渲染
val timeSinceLastRender = now - lastRenderTimeMs
if (timeSinceLastRender < renderIntervalMs - renderJitterMs) {
return@Runnable
}
// 低水位启动渲染逻辑
if (!renderStarted) {
if (outputFrameQueue.size >= (BUFFER_QUEUE_CAPACITY * 0.15).toInt()) {
renderStarted = true
Log.i(TAG, "[Render] 渲染启动outputFrameQueue已达低水位: ${outputFrameQueue.size}")
} else {
// 未达到低水位前不渲染
return@Runnable
}
}
val frame = outputFrameQueue.poll()
if (frame != null) {
frame.codec.releaseOutputBuffer(frame.bufferIndex, true)
latestRenderedTimestampMs = System.currentTimeMillis()
lastRenderTimeMs = now
latestRenderedTimestampMs = now
renderedFrameCount++
if (!hasNotifiedFlutter) {
mainHandler.post { onFrameRendered() }
hasNotifiedFlutter = true
}
// Log.d(TAG, "[Render] 渲染: bufferIdx=${frame.bufferIndex}, pts=${frame.timestampUs}, " +
// "当前outputFrameQueue=${outputFrameQueue.size}")
} else {
// Log.w(TAG, "[Render] 渲染空转无帧可渲染当前outputFrameQueue=${outputFrameQueue.size}")
}
// 若outputFrameQueue为空跳过本次渲染实现线性调度
} catch (e: Exception) {
Log.e(TAG, "[RenderTask] Exception", e)
}
}
// 固定20fps渲染50ms间隔
scheduler.scheduleAtFixedRate(renderTask, 0, 50, java.util.concurrent.TimeUnit.MILLISECONDS)
// 使用更短的调度间隔,但在任务中控制实际渲染间隔
scheduler.scheduleAtFixedRate(renderTask, 0, renderIntervalMs/2, TimeUnit.MILLISECONDS)
}
// endregion
@ -253,7 +323,10 @@ class VideoDecoder(
refIFrameSeq: Int?
): Boolean {
if (!running || mediaCodec == null) return false
if (!frameSeqSet.add(frameSeq)) return false // 防止重复帧
// if (!frameSeqSet.add(frameSeq)) {
// Log.w(TAG, "[decodeFrame] 丢弃重复帧: type=$frameType, seq=$frameSeq, refI=$refIFrameSeq, ts=$timestamp")
// return false // 防止重复帧
// }
// 2. 初始化起点
if (timestampBaseMs == null) {
synchronized(this) {
@ -270,7 +343,7 @@ class VideoDecoder(
// 3. decodeFrame延迟丢弃判断用系统时间
val now = System.currentTimeMillis()
if (absTimestamp < now - MAX_ALLOWED_DELAY_MS) {
Log.w(TAG, "[decodeFrame] Drop frame due to delay: absFrameTs=$absTimestamp, now=$now, maxDelay=$MAX_ALLOWED_DELAY_MS")
Log.w(TAG, "[decodeFrame] 丢弃延迟帧: type=$frameType, seq=$frameSeq, absTs=$absTimestamp, now=$now, maxDelay=$MAX_ALLOWED_DELAY_MS")
return false
}
// ===== 帧重排序缓冲区机制 =====
@ -278,36 +351,43 @@ class VideoDecoder(
if (frameType == 0) { // I帧
receivedIFrames.add(frameSeq)
lastIFrameSeq = frameSeq
// Log.d(TAG, "[reorder] I帧到达: seq=$frameSeq, 当前缓存P帧数=${reorderBuffer.size}")
// I帧直接入解码队列
inputFrameQueue.offer(FrameData(frameData, frameType, timestamp, frameSeq, refIFrameSeq), 50, TimeUnit.MILLISECONDS)
inputFrameQueue.offer(FrameData(frameData, frameType, timestamp, frameSeq, refIFrameSeq), 150, TimeUnit.MILLISECONDS)
// 检查缓冲区入队所有依赖于该I帧的P帧
val readyPFrames = reorderBuffer.values.filter { it.refIFrameSeq == frameSeq }
.sortedBy { it.frameSeq }
// Log.d(TAG, "[reorder] I帧释放P帧: 依赖seq=$frameSeq, 释放P帧数=${readyPFrames.size}")
for (pFrame in readyPFrames) {
inputFrameQueue.offer(pFrame, 50, TimeUnit.MILLISECONDS)
inputFrameQueue.offer(pFrame, 150, TimeUnit.MILLISECONDS)
reorderBuffer.remove(pFrame.frameSeq)
}
// 清理过期P帧如缓冲区过大
if (reorderBuffer.size > MAX_REORDER_BUFFER_SIZE) {
val toRemove = reorderBuffer.keys.sorted().take(reorderBuffer.size - MAX_REORDER_BUFFER_SIZE)
Log.w(TAG, "[reorder] 缓冲区溢出清理P帧: 清理数=${toRemove.size}")
toRemove.forEach { reorderBuffer.remove(it) }
}
// Log.d(TAG, "[decodeFrame] 入队I帧: seq=$frameSeq, ts=$timestamp, 当前inputFrameQueue=${inputFrameQueue.size}")
return true
} else { // P帧
val lastI = lastIFrameSeq
// 只有依赖的I帧已收到才允许入队否则暂存
if (refIFrameSeq != null && receivedIFrames.contains(refIFrameSeq)) {
inputFrameQueue.offer(FrameData(frameData, frameType, timestamp, frameSeq, refIFrameSeq), 50, TimeUnit.MILLISECONDS)
inputFrameQueue.offer(FrameData(frameData, frameType, timestamp, frameSeq, refIFrameSeq), 150, TimeUnit.MILLISECONDS)
// Log.d(TAG, "[decodeFrame] 入队P帧: seq=$frameSeq, refI=$refIFrameSeq, ts=$timestamp, 当前inputFrameQueue=${inputFrameQueue.size}")
return true
} else {
// 暂存到重排序缓冲区
reorderBuffer[frameSeq] = FrameData(frameData, frameType, timestamp, frameSeq, refIFrameSeq)
Log.d(TAG, "[reorder] P帧缓存: seq=$frameSeq, refI=$refIFrameSeq, 当前缓存=${reorderBuffer.size}")
// 控制缓冲区大小
if (reorderBuffer.size > MAX_REORDER_BUFFER_SIZE) {
val toRemove = reorderBuffer.keys.sorted().take(reorderBuffer.size - MAX_REORDER_BUFFER_SIZE)
Log.w(TAG, "[reorder] 缓冲区溢出清理P帧: 清理数=${toRemove.size}")
toRemove.forEach { reorderBuffer.remove(it) }
}
Log.w(TAG, "[decodeFrame] P-frame cached: frameSeq=$frameSeq, refIFrameSeq=$refIFrameSeq, waiting for I-frame.")
Log.w(TAG, "[decodeFrame] P帧暂存: seq=$frameSeq, refI=$refIFrameSeq, 等待I帧")
return false
}
}
@ -334,33 +414,25 @@ class VideoDecoder(
} catch (_: Exception) {}
}
/**
* 计算最近N帧的平均解码帧率fps
*/
private fun calculateDecodeFps(): Double {
decodeTimestampLock.withLock {
if (decodeTimestampQueue.size < 2) return renderFps.toDouble()
val first = decodeTimestampQueue.first()
val last = decodeTimestampQueue.last()
val frameCount = decodeTimestampQueue.size - 1
val durationMs = (last - first).coerceAtLeast(1L)
return frameCount * 1000.0 / durationMs
}
}
// private fun calculateDecodeFps(): Double {
// decodeTimestampLock.withLock {
// if (decodeTimestampQueue.size < 2) return renderFps.toDouble()
// val first = decodeTimestampQueue.first()
// val last = decodeTimestampQueue.last()
// val frameCount = decodeTimestampQueue.size - 1
// val durationMs = (last - first).coerceAtLeast(1L)
// return frameCount * 1000.0 / durationMs
// }
// }
/**
* EMA平滑更新渲染帧率
* @param measuredFps 当前测得的解码帧率
* @return 平滑后的渲染帧率取整
*/
private fun updateSmoothedFps(measuredFps: Double): Int {
// measuredFps边界保护
val safeFps = measuredFps.coerceIn(minFps.toDouble(), maxFps.toDouble())
val targetFps = alpha * safeFps + (1 - alpha) * smoothedFps
val delta = targetFps - smoothedFps
val step = delta.coerceIn(-maxStep, maxStep)
smoothedFps = (smoothedFps + step).coerceIn(minFps.toDouble(), maxFps.toDouble())
return smoothedFps.toInt()
}
// private fun updateSmoothedFps(measuredFps: Double): Int {
// // measuredFps边界保护
// val safeFps = measuredFps.coerceIn(minFps.toDouble(), maxFps.toDouble())
// val targetFps = alpha * safeFps + (1 - alpha) * smoothedFps
// val delta = targetFps - smoothedFps
// val step = delta.coerceIn(-maxStep, maxStep)
// smoothedFps = (smoothedFps + step).coerceIn(minFps.toDouble(), maxFps.toDouble())
// return smoothedFps.toInt()
// }
// endregion
}

View File

@ -1,4 +1,4 @@
org.gradle.jvmargs=-Xmx4G
android.useAndroidX=true
android.enableJetifier=true
org.gradle.java.home=/Library/Java/JavaVirtualMachines/jdk-17.0.1.jdk/Contents/Home
#org.gradle.java.home=C:/Users/liyi/other/jdk-17.0.1

View File

@ -42,9 +42,11 @@ public class VideoDecodePlugin: NSObject, FlutterPlugin, FlutterTexture {
let width = args["width"] as? Int,
let height = args["height"] as? Int,
let codecType = args["codecType"] as? String else {
print("[VideoDecodePlugin][错误] 参数解析失败:\(String(describing: call.arguments))")
result(FlutterError(code: "INVALID_ARGS", message: "参数错误", details: nil))
return
}
//
decoder?.release()
decoder = nil
@ -52,13 +54,17 @@ public class VideoDecodePlugin: NSObject, FlutterPlugin, FlutterTexture {
textureRegistry?.unregisterTexture(tid)
textureId = nil
}
// Flutter
guard let registry = textureRegistry else {
print("[VideoDecodePlugin][错误] 无法获取纹理注册表")
result(FlutterError(code: "NO_TEXTURE_REGISTRY", message: "无法获取纹理注册表", details: nil))
return
}
let textureId = registry.register(self)
self.textureId = textureId
//
let decoder = VideoDecoder(width: width, height: height, codecType: codecType)
self.decoder = decoder
@ -79,32 +85,94 @@ public class VideoDecodePlugin: NSObject, FlutterPlugin, FlutterTexture {
result(textureId)
}
/// NALU
/// NALU
private func stripStartCode(_ data: Data) -> Data {
let originalLen = data.count
let naluType: UInt8 = {
if data.count > 4 && data[0] == 0x00 && data[1] == 0x00 && data[2] == 0x00 && data[3] == 0x01 {
return data[4] & 0x1F
} else if data.count > 3 && data[0] == 0x00 && data[1] == 0x00 && data[2] == 0x01 {
return data[3] & 0x1F
}
return 0
}()
var stripped: Data = data
if data.count > 4 && data[0] == 0x00 && data[1] == 0x00 && data[2] == 0x00 && data[3] == 0x01 {
stripped = data.subdata(in: 4..<data.count)
} else if data.count > 3 && data[0] == 0x00 && data[1] == 0x00 && data[2] == 0x01 {
stripped = data.subdata(in: 3..<data.count)
}
let strippedLen = stripped.count
let strippedType: UInt8 = stripped.count > 0 ? (stripped[0] & 0x1F) : 0
if strippedLen < 3 || (strippedType != 7 && strippedType != 8) {
print("[VideoDecodePlugin][警告] strip后NALU长度或类型异常type=", strippedType, "len=", strippedLen)
}
//
return stripped
}
// side_data
private func checkNaluForSideData(_ nalu: Data, naluType: UInt8) -> Bool {
let maxSize = naluType == 5 ? 150_000 : 30_000 // I150KBP30KB
if nalu.count > maxSize {
print("[VideoDecodePlugin][警告] NALU长度异常可能包含side_datatype=\(naluType)len=\(nalu.count)")
return true
}
return false
}
// NALU
private func extractFirstValidNalu(_ nalu: Data) -> Data {
// 34
var start = -1
var startCodeLength = 0
// 4
if nalu.count >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00 && nalu[2] == 0x00 && nalu[3] == 0x01 {
start = 0
startCodeLength = 4
}
// 3
else if nalu.count >= 3 && nalu[0] == 0x00 && nalu[1] == 0x00 && nalu[2] == 0x01 {
start = 0
startCodeLength = 3
}
//
else {
for i in 0..<(nalu.count - 4) {
if nalu[i] == 0x00 && nalu[i + 1] == 0x00 {
if nalu[i + 2] == 0x00 && nalu[i + 3] == 0x01 {
start = i
startCodeLength = 4
break
} else if nalu[i + 2] == 0x01 {
start = i
startCodeLength = 3
break
}
}
}
}
if start == -1 {
print("[VideoDecodePlugin][警告] NALU无AnnexB起始码丢弃该帧")
return Data()
}
let searchRange = (start + startCodeLength)..<nalu.count
var end = nalu.count
// 34
for i in searchRange.lowerBound..<(nalu.count - 3) {
if nalu[i] == 0x00 && nalu[i + 1] == 0x00 {
if i + 3 < nalu.count && nalu[i + 2] == 0x00 && nalu[i + 3] == 0x01 {
end = i
break
} else if nalu[i + 2] == 0x01 {
end = i
break
}
}
}
let extractedNalu = nalu[start..<end]
// NALU
if extractedNalu.count >= startCodeLength + 1 {
let naluType = extractedNalu[startCodeLength] & 0x1F
if checkNaluForSideData(extractedNalu, naluType: naluType) {
return Data()
}
}
return extractedNalu
}
///
private func handleDecodeFrame(call: FlutterMethodCall, result: @escaping FlutterResult) {
guard let args = call.arguments as? [String: Any],
@ -128,30 +196,32 @@ public class VideoDecodePlugin: NSObject, FlutterPlugin, FlutterTexture {
return 0
}()
print("[VideoDecodePlugin][调试] handleDecodeFrame: frameType=\(frameType), naluType=\(naluType), cachedSpsLen=\(cachedSps?.count ?? 0), cachedPpsLen=\(cachedPps?.count ?? 0)")
// SPS/PPS
// SPS/PPS
if naluType == 7 { // SPS
//
cachedSps = stripStartCode(data)
result(true)
return
} else if naluType == 8 { // PPS
//
cachedPps = stripStartCode(data)
result(true)
return
} else if naluType == 5 { // IDR/I
// NALU
// NALU
let firstNalu = extractFirstValidNalu(data)
if firstNalu.isEmpty { return }
print("[VideoDecodePlugin] 发送I帧, 长度: \(firstNalu.count), 头部: \(firstNalu.prefix(8).map { String(format: "%02X", $0) }.joined(separator: " ")), cachedSps长度: \(cachedSps?.count ?? 0), cachedPps长度: \(cachedPps?.count ?? 0)")
if firstNalu.isEmpty {
result(false)
return
}
decoder?.decodeFrame(frameData: firstNalu, frameType: frameType, timestamp: Int64(timestamp), frameSeq: frameSeq, refIFrameSeq: refIFrameSeq, sps: cachedSps, pps: cachedPps)
} else {
// NALU
// NALU
let firstNalu = extractFirstValidNalu(data)
if firstNalu.isEmpty { return }
print("[VideoDecodePlugin] 发送P/B帧, 长度: \(firstNalu.count), 头部: \(firstNalu.prefix(8).map { String(format: "%02X", $0) }.joined(separator: " "))")
if firstNalu.isEmpty {
result(false)
return
}
decoder?.decodeFrame(frameData: firstNalu, frameType: frameType, timestamp: Int64(timestamp), frameSeq: frameSeq, refIFrameSeq: refIFrameSeq)
}
result(true)
@ -182,28 +252,4 @@ public class VideoDecodePlugin: NSObject, FlutterPlugin, FlutterTexture {
}
return nil
}
// side_data
private func checkNaluForSideData(_ nalu: Data, naluType: UInt8) -> Bool {
if (naluType == 5 && nalu.count > 10000) || (naluType != 7 && naluType != 8 && nalu.count > 10000) {
print("[VideoDecodePlugin][警告] NALU长度异常可能包含side_datatype=\(naluType)len=\(nalu.count)")
return true
}
return false
}
// NALU
private func extractFirstValidNalu(_ nalu: Data) -> Data {
guard let start = nalu.range(of: Data([0x00, 0x00, 0x00, 0x01]))?.lowerBound else {
print("[VideoDecodePlugin][警告] NALU无AnnexB起始码丢弃该帧")
return Data()
}
let searchRange = (start+4)..<nalu.count
if let next = nalu[searchRange].range(of: Data([0x00, 0x00, 0x00, 0x01]))?.lowerBound {
let end = searchRange.lowerBound + next
return nalu[start..<end]
} else {
return nalu[start..<nalu.count]
}
}
}

View File

@ -35,49 +35,50 @@ class VideoDecoder {
///
private var frameSeqSet = Set<Int>()
///
private let maxAllowedDelayMs: Int = 350
private let maxAllowedDelayMs: Int64 = 750 // Android
///
private var timestampBaseMs: Int64?
///
private var firstFrameRelativeTimestamp: Int64?
// ====== ======
// ====== ======
/// 线
private let inputQueue = DispatchQueue(label: "video_decode_plugin.input.queue", attributes: .concurrent)
private var inputBuffer: [(frameData: Data, frameType: Int, timestamp: Int64, frameSeq: Int, refIFrameSeq: Int?, sps: Data?, pps: Data?)] = []
private let inputBufferSemaphore = DispatchSemaphore(value: 1)
private let inputBufferMaxCount = 15
private let inputBufferMaxCount = 100 // Android
/// 线
private let outputQueue = DispatchQueue(label: "video_decode_plugin.output.queue", attributes: .concurrent)
private var outputBuffer: [(pixelBuffer: CVPixelBuffer, timestamp: Int64)] = []
private let outputBufferSemaphore = DispatchSemaphore(value: 1)
private let outputBufferMaxCount = 15
/// 线
private var renderThread: Thread?
private let outputBufferMaxCount = 100 // Android
///
private var renderTimer: DispatchSourceTimer?
/// 线
private var renderThreadRunning = false
///
private var hasNotifiedFlutter = false
///
private var renderFps: Int = 15
/// EMA
private var smoothedFps: Double = 15.0
/// EMA
private let alpha: Double = 0.2
///
private let minFps: Double = 8.0
///
private let maxFps: Double = 30.0
///
private let maxStep: Double = 2.0
///
private var renderedTimestamps: [Int64] = [] // ms
///
private let renderedTimestampsMaxCount = 20
///
private var renderedFrameCount = 0
/// N
private let fpsAdjustInterval = 10
private var renderFps: Int = 20 // Android
///
private var renderIntervalMs: Int64 = 0
///
private let renderJitterMs: Int64 = 2
///
private var lastRenderTimeMs: Int64 = 0
///
///
private let renderLowWaterMark = 15
// ====== ======
///
private var reorderBuffer: [Int: (frameData: Data, frameType: Int, timestamp: Int64, frameSeq: Int, refIFrameSeq: Int?, sps: Data?, pps: Data?)] = [:]
/// I
private var receivedIFrames = Set<Int>()
///
private let reorderLock = NSLock()
///
private let maxReorderBufferSize = 100 //
/// CVPixelBuffer
var onFrameDecoded: ((CVPixelBuffer, Int64) -> Void)? = { _, _ in }
@ -87,7 +88,9 @@ class VideoDecoder {
self.width = width
self.height = height
self.codecType = CodecType(rawValue: codecType.lowercased()) ?? .h264
startRenderThread()
self.renderIntervalMs = Int64(1000.0 / Double(renderFps))
startRenderTimer()
print("[VideoDecoder] 初始化解码器: width=\(width), height=\(height)")
}
// ====== ======
@ -95,7 +98,8 @@ class VideoDecoder {
private func enqueueInput(_ item: (Data, Int, Int64, Int, Int?, Data?, Data?)) {
inputQueue.async(flags: .barrier) {
if self.inputBuffer.count >= self.inputBufferMaxCount {
self.inputBuffer.removeFirst()
self.inputBuffer.removeFirst() //
print("[VideoDecoder][警告] 输入缓冲区满,丢弃最旧帧")
}
self.inputBuffer.append(item)
}
@ -115,7 +119,8 @@ class VideoDecoder {
private func enqueueOutput(_ item: (CVPixelBuffer, Int64)) {
outputQueue.async(flags: .barrier) {
if self.outputBuffer.count >= self.outputBufferMaxCount {
self.outputBuffer.removeFirst()
self.outputBuffer.removeFirst() //
print("[VideoDecoder][警告] 输出缓冲区满,丢弃最旧帧")
}
self.outputBuffer.append(item)
}
@ -130,72 +135,136 @@ class VideoDecoder {
}
return item
}
// ====== 线 ======
/// 线FlutterEMA
private func startRenderThread() {
renderThreadRunning = true
renderThread = Thread { [weak self] in
guard let self = self else { return }
while self.renderThreadRunning {
let frameIntervalMs = Int(1000.0 / self.smoothedFps)
let loopStart = Date().timeIntervalSince1970 * 1000.0
if let (pixelBuffer, timestamp) = self.dequeueOutput() {
// Flutter
DispatchQueue.main.async {
self.onFrameDecoded?(pixelBuffer, timestamp)
}
// Flutter
if !self.hasNotifiedFlutter {
self.hasNotifiedFlutter = true
// onFrameRendered
}
//
self.renderedTimestamps.append(Int64(Date().timeIntervalSince1970 * 1000))
if self.renderedTimestamps.count > self.renderedTimestampsMaxCount {
self.renderedTimestamps.removeFirst()
}
self.renderedFrameCount += 1
if self.renderedFrameCount % self.fpsAdjustInterval == 0 {
let measuredFps = self.calculateDecodeFps()
let newFps = self.updateSmoothedFps(measuredFps)
self.renderFps = newFps
}
// ====== ======
///
private func handleFrameReordering(frameData: Data, frameType: Int, timestamp: Int64, frameSeq: Int, refIFrameSeq: Int?, sps: Data?, pps: Data?) -> Bool {
reorderLock.lock()
defer { reorderLock.unlock() }
// 1.
let now = Int64(Date().timeIntervalSince1970 * 1000)
let base = timestampBaseMs ?? 0
let firstRel = firstFrameRelativeTimestamp ?? 0
let absTimestamp = base + (timestamp - firstRel)
// if absTimestamp < now - maxAllowedDelayMs {
// print("[VideoDecoder][] : type=\(frameType), seq=\(frameSeq), delay=\(now - absTimestamp)ms")
// return false
// }
// 2.
if frameType == 0 { // I
receivedIFrames.insert(frameSeq)
lastIFrameSeq = frameSeq
// I
enqueueInput((frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps))
// IP
let readyPFrames = reorderBuffer.values
.filter { $0.refIFrameSeq == frameSeq }
.sorted { $0.frameSeq < $1.frameSeq }
for pFrame in readyPFrames {
enqueueInput(pFrame)
reorderBuffer.removeValue(forKey: pFrame.frameSeq)
}
if !readyPFrames.isEmpty {
print("[VideoDecoder] I帧\(frameSeq)释放\(readyPFrames.count)个P帧")
}
//
if reorderBuffer.count > maxReorderBufferSize {
let toRemove = reorderBuffer.keys.sorted().prefix(reorderBuffer.count - maxReorderBufferSize)
for seq in toRemove {
reorderBuffer.removeValue(forKey: seq)
}
//
let loopCost = Int(Date().timeIntervalSince1970 * 1000.0 - loopStart)
let sleepMs = frameIntervalMs - loopCost
if sleepMs > 0 {
Thread.sleep(forTimeInterval: Double(sleepMs) / 1000.0)
print("[VideoDecoder][警告] 重排序缓冲区溢出,清理\(toRemove.count)个P帧")
}
return true
} else { // P
// P
if let refIFrameSeq = refIFrameSeq, receivedIFrames.contains(refIFrameSeq) {
// I
enqueueInput((frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps))
return true
} else {
// IP
reorderBuffer[frameSeq] = (frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps)
print("[VideoDecoder] P帧\(frameSeq)缓存等待I帧\(refIFrameSeq ?? -1)")
//
if reorderBuffer.count > maxReorderBufferSize {
let toRemove = reorderBuffer.keys.sorted().prefix(reorderBuffer.count - maxReorderBufferSize)
for seq in toRemove {
reorderBuffer.removeValue(forKey: seq)
}
print("[VideoDecoder][警告] 重排序缓冲区溢出,清理\(toRemove.count)个P帧")
}
return false
}
}
}
// ====== ======
/// Flutter
private func startRenderTimer() {
renderThreadRunning = true
let timer = DispatchSource.makeTimerSource(queue: DispatchQueue.global())
timer.schedule(deadline: .now(), repeating: .milliseconds(Int(renderIntervalMs / 2)))
timer.setEventHandler { [weak self] in
guard let self = self else { return }
let now = Int64(Date().timeIntervalSince1970 * 1000)
//
let timeSinceLastRender = now - self.lastRenderTimeMs
if timeSinceLastRender < self.renderIntervalMs - self.renderJitterMs {
return
}
//
var outputCount = 0
self.outputQueue.sync { outputCount = self.outputBuffer.count }
if outputCount < self.renderLowWaterMark {
//
return
}
if let (pixelBuffer, timestamp) = self.dequeueOutput() {
//
let now = Int64(Date().timeIntervalSince1970 * 1000)
let base = timestampBaseMs ?? 0
let firstRel = firstFrameRelativeTimestamp ?? 0
let absTimestamp = base + (timestamp - firstRel)
// if absTimestamp < now - self.maxAllowedDelayMs {
// print("[VideoDecoder][] : delay=\(now - absTimestamp)ms")
// return
// }
DispatchQueue.main.async {
self.onFrameDecoded?(pixelBuffer, timestamp)
}
self.lastRenderTimeMs = now
if !self.hasNotifiedFlutter {
self.hasNotifiedFlutter = true
}
}
}
renderThread?.start()
timer.resume()
renderTimer = timer
}
/// 线
private func stopRenderThread() {
///
private func stopRenderTimer() {
renderThreadRunning = false
renderThread?.cancel()
renderThread = nil
}
// ====== EMA ======
/// N
private func calculateDecodeFps() -> Double {
guard renderedTimestamps.count >= 2 else { return smoothedFps }
let first = renderedTimestamps.first!
let last = renderedTimestamps.last!
let frameCount = renderedTimestamps.count - 1
let durationMs = max(last - first, 1)
return Double(frameCount) * 1000.0 / Double(durationMs)
}
/// EMA
private func updateSmoothedFps(_ measuredFps: Double) -> Int {
let safeFps = min(max(measuredFps, minFps), maxFps)
let targetFps = alpha * safeFps + (1 - alpha) * smoothedFps
let delta = targetFps - smoothedFps
let step = min(max(delta, -maxStep), maxStep)
smoothedFps = min(max(smoothedFps + step, minFps), maxFps)
return Int(smoothedFps)
renderTimer?.cancel()
renderTimer = nil
}
/// I
private func setupSession(sps: Data?, pps: Data?) -> Bool {
//
@ -206,9 +275,10 @@ class VideoDecoder {
formatDesc = nil
isSessionReady = false
guard let sps = sps, let pps = pps else {
print("[VideoDecoder] 缺少SPS/PPS无法初始化解码会话")
print("[VideoDecoder][错误] 缺少SPS/PPS无法初始化解码会话")
return false
}
// SPS/PPS
let spsType: UInt8 = sps.count > 0 ? (sps[0] & 0x1F) : 0
let ppsType: UInt8 = pps.count > 0 ? (pps[0] & 0x1F) : 0
@ -220,6 +290,7 @@ class VideoDecoder {
print("[VideoDecoder][错误] PPS内容异常len=\(pps.count), type=\(ppsType)")
return false
}
var success = false
sps.withUnsafeBytes { spsPtr in
pps.withUnsafeBytes { ppsPtr in
@ -237,7 +308,7 @@ class VideoDecoder {
formatDescriptionOut: &formatDesc
)
if status != noErr {
print("[VideoDecoder] 创建FormatDescription失败: \(status)")
print("[VideoDecoder][错误] 创建FormatDescription失败: \(status)")
success = false
} else {
success = true
@ -252,7 +323,7 @@ class VideoDecoder {
// 线
decoder.enqueueOutput((pixelBuffer, Int64(pts.seconds * 1000)))
} else {
print("[VideoDecoder] 解码回调失败, status=\(status)")
print("[VideoDecoder][错误] 解码回调失败: \(status)")
}
},
decompressionOutputRefCon: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())
@ -261,8 +332,10 @@ class VideoDecoder {
kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
kCVPixelBufferWidthKey: width,
kCVPixelBufferHeightKey: height,
kCVPixelBufferOpenGLESCompatibilityKey: true
kCVPixelBufferOpenGLESCompatibilityKey: true,
kCVPixelBufferIOSurfacePropertiesKey: [:]
]
let status2 = VTDecompressionSessionCreate(
allocator: kCFAllocatorDefault,
formatDescription: formatDesc!,
@ -272,7 +345,7 @@ class VideoDecoder {
decompressionSessionOut: &decompressionSession
)
if status2 != noErr {
print("[VideoDecoder] 创建解码会话失败: \(status2)")
print("[VideoDecoder][错误] 创建解码会话失败: \(status2)")
return false
}
isSessionReady = true
@ -282,38 +355,91 @@ class VideoDecoder {
///
func decodeFrame(frameData: Data, frameType: Int, timestamp: Int64, frameSeq: Int, refIFrameSeq: Int?, sps: Data? = nil, pps: Data? = nil) {
enqueueInput((frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps))
// 线
// 1.
if timestampBaseMs == nil {
timestampBaseMs = Int64(Date().timeIntervalSince1970 * 1000)
firstFrameRelativeTimestamp = timestamp
print("[VideoDecoder] 设置时间戳基准: base=\(timestampBaseMs!), firstRel=\(firstFrameRelativeTimestamp!)")
}
// 2.
if !handleFrameReordering(frameData: frameData, frameType: frameType, timestamp: timestamp, frameSeq: frameSeq, refIFrameSeq: refIFrameSeq, sps: sps, pps: pps) {
return
}
// 3. inputQueue
decodeQueue.async { [weak self] in
guard let self = self else { return }
guard let (frameData, frameType, timestamp, frameSeq, refIFrameSeq, sps, pps) = self.dequeueInput() else { return }
if !self.isSessionReady, let sps = sps, let pps = pps {
guard self.setupSession(sps: sps, pps: pps) else { return }
}
guard let session = self.decompressionSession else { return }
guard frameData.count > 4 else { return }
var avccData = frameData
let naluLen = UInt32(frameData.count - 4).bigEndian
if avccData.count >= 4 {
avccData.replaceSubrange(0..<4, with: withUnsafeBytes(of: naluLen) { Data($0) })
guard frameData.count > 3 else { return }
// AnnexB
var startCodeSize = 0
var naluData: Data
if frameData.count >= 4 && frameData[0] == 0x00 && frameData[1] == 0x00 && frameData[2] == 0x00 && frameData[3] == 0x01 {
startCodeSize = 4
naluData = frameData.subdata(in: 4..<frameData.count)
} else if frameData.count >= 3 && frameData[0] == 0x00 && frameData[1] == 0x00 && frameData[2] == 0x01 {
startCodeSize = 3
naluData = frameData.subdata(in: 3..<frameData.count)
} else {
return
print("[VideoDecoder][警告] 未找到起始码")
naluData = frameData
}
// AVCC
let naluLength = UInt32(naluData.count).bigEndian
var avccData = Data(capacity: naluData.count + 4)
withUnsafeBytes(of: naluLength) { ptr in
avccData.append(ptr.baseAddress!.assumingMemoryBound(to: UInt8.self), count: 4)
}
avccData.append(naluData)
var blockBuffer: CMBlockBuffer?
let status = CMBlockBufferCreateWithMemoryBlock(
allocator: kCFAllocatorDefault,
memoryBlock: UnsafeMutableRawPointer(mutating: (avccData as NSData).bytes),
memoryBlock: nil,
blockLength: avccData.count,
blockAllocator: kCFAllocatorNull,
blockAllocator: nil,
customBlockSource: nil,
offsetToData: 0,
dataLength: avccData.count,
flags: 0,
flags: kCMBlockBufferAssureMemoryNowFlag,
blockBufferOut: &blockBuffer
)
if status != kCMBlockBufferNoErr { return }
if status != kCMBlockBufferNoErr {
print("[VideoDecoder][错误] 创建BlockBuffer失败: \(status)")
return
}
// BlockBuffer
if let blockBuffer = blockBuffer {
let status2 = avccData.withUnsafeBytes { ptr in
CMBlockBufferReplaceDataBytes(
with: ptr.baseAddress!,
blockBuffer: blockBuffer,
offsetIntoDestination: 0,
dataLength: avccData.count
)
}
if status2 != kCMBlockBufferNoErr {
print("[VideoDecoder][错误] 复制数据到BlockBuffer失败: \(status2)")
return
}
}
var sampleBuffer: CMSampleBuffer?
var timing = CMSampleTimingInfo(duration: .invalid, presentationTimeStamp: CMTime(value: timestamp, timescale: 1000), decodeTimeStamp: .invalid)
var timing = CMSampleTimingInfo(
duration: .invalid,
presentationTimeStamp: CMTime(value: timestamp, timescale: 1000),
decodeTimeStamp: .invalid
)
let status2 = CMSampleBufferCreate(
allocator: kCFAllocatorDefault,
dataBuffer: blockBuffer,
@ -328,8 +454,12 @@ class VideoDecoder {
sampleSizeArray: [avccData.count],
sampleBufferOut: &sampleBuffer
)
if status2 != noErr { return }
let decodeFlags: VTDecodeFrameFlags = []
if status2 != noErr {
print("[VideoDecoder][错误] 创建SampleBuffer失败: \(status2)")
return
}
let decodeFlags: VTDecodeFrameFlags = [._EnableAsynchronousDecompression]
var infoFlags = VTDecodeInfoFlags()
let status3 = VTDecompressionSessionDecodeFrame(
session,
@ -339,14 +469,19 @@ class VideoDecoder {
infoFlagsOut: &infoFlags
)
if status3 != noErr {
print("[VideoDecoder] 解码失败: \(status3)")
print("[VideoDecoder][错误] 解码失败: \(status3)")
if status3 == -6661 {
print(" - 错误类型: kVTInvalidSessionErr (解码会话无效)")
print(" - 会话状态: \(self.isSessionReady ? "就绪" : "未就绪")")
print(" - formatDesc: \(self.formatDesc != nil ? "有效" : "无效")")
}
}
}
}
///
func release() {
stopRenderThread()
stopRenderTimer()
decodeQueue.sync {
if let session = decompressionSession {
VTDecompressionSessionInvalidate(session)
@ -356,6 +491,12 @@ class VideoDecoder {
isSessionReady = false
frameSeqSet.removeAll()
lastIFrameSeq = nil
//
reorderLock.lock()
reorderBuffer.removeAll()
receivedIFrames.removeAll()
reorderLock.unlock()
}
inputQueue.async(flags: .barrier) { self.inputBuffer.removeAll() }
outputQueue.async(flags: .barrier) { self.outputBuffer.removeAll() }