From 4cdd15d4555e8701e090a350706ac0a3d57be3f7 Mon Sep 17 00:00:00 2001 From: liyi Date: Wed, 18 Jun 2025 11:22:56 +0800 Subject: [PATCH] =?UTF-8?q?feat:v2=E7=89=88=E6=9C=AC=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../video_decode_plugin/VideoDecoder.kt | 159 +++++++++--------- lib/video_decode_plugin.dart | 4 + 2 files changed, 87 insertions(+), 76 deletions(-) diff --git a/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoder.kt b/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoder.kt index c622cc0..7357f7b 100644 --- a/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoder.kt +++ b/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoder.kt @@ -19,6 +19,7 @@ import java.util.Collections import java.util.concurrent.ConcurrentHashMap import android.os.SystemClock import java.util.ArrayDeque +import java.util.concurrent.Executors /** * 视频解码器核心类。 @@ -43,12 +44,12 @@ class VideoDecoder( width: Int, height: Int, codecType: String, - private val onFrameRendered: () -> Unit + private val onFrameRendered: () -> Unit, ) { companion object { private const val TAG = "VideoDecoder" private const val TIMEOUT_US = 10000L - private const val INPUT_BUFFER_QUEUE_CAPACITY = 15 // 输入缓冲区容量 + private const val INPUT_BUFFER_QUEUE_CAPACITY = 50 // 输入缓冲区容量 } // region 成员变量定义 @@ -64,37 +65,49 @@ class VideoDecoder( private val frameSeqSet = Collections.newSetFromMap(ConcurrentHashMap()) // 防止重复帧入队 // 解码输出缓冲区，容量为100帧 - private val outputFrameQueue = LinkedBlockingQueue(15) + private val outputFrameQueue = LinkedBlockingQueue(50) // 渲染线程控制 - @Volatile private var renderThreadRunning = true - private var renderThread: Thread? = null + // 定时渲染调度器 + private var scheduler = Executors.newSingleThreadScheduledExecutor() // 主线程Handler，用于安全切换onFrameRendered到主线程 private val mainHandler = Handler(Looper.getMainLooper()) // 渲染帧率（fps），可由外部控制，默认18 - @Volatile var renderFps: Int = 15 + @Volatile + var renderFps: Int = 20 // 兜底：记录最近一次I帧的frameSeq，P/B帧依赖校验 - @Volatile private var lastIFrameSeq: Int? = null + @Volatile + private var lastIFrameSeq: Int? = null // 解码输出帧时间戳队列（用于动态帧率统计和平滑） private val decodeTimestampQueue = ArrayDeque(20) // 最多保存20帧时间戳 private val decodeTimestampLock = ReentrantLock() // 线程安全保护 // EMA平滑参数 - @Volatile private var smoothedFps: Double = 15.0 // 平滑后的渲染帧率 + @Volatile + private var smoothedFps: Double = 25.0 // 平滑后的渲染帧率 private val alpha = 0.2 // EMA平滑系数，越大响应越快 private val minFps = 8 // 渲染帧率下限，防止过低 private val maxFps = 30 // 渲染帧率上限，防止过高 private val maxStep = 2.0 // 单次最大调整幅度，防止突变 // 1. 新增成员变量 - @Volatile private var latestRenderedTimestampMs: Long? = null - private val MAX_ALLOWED_DELAY_MS = 350 // 最大允许延迟，单位毫秒 - @Volatile private var timestampBaseMs: Long? = null - @Volatile private var firstFrameRelativeTimestamp: Long? = null + @Volatile + private var latestRenderedTimestampMs: Long? = null + private val MAX_ALLOWED_DELAY_MS = 550 // 最大允许延迟，单位毫秒 + @Volatile + private var timestampBaseMs: Long? = null + @Volatile + private var firstFrameRelativeTimestamp: Long? = null + + // 帧重排序缓冲区及I帧记录 + private val reorderBuffer = mutableMapOf() // key: frameSeq + private val receivedIFrames = mutableSetOf() // 已收到的I帧frameSeq + private val reorderLock = ReentrantLock() // 线程安全 + private val MAX_REORDER_BUFFER_SIZE = 50 // 输入帧结构体 private data class FrameData( @@ -102,7 +115,7 @@ class VideoDecoder( val frameType: Int, val timestamp: Long, val frameSeq: Int, - val refIFrameSeq: Int? + val refIFrameSeq: Int?, ) // 解码后帧结构体，显式携带时间戳（单位：微秒） @@ -110,7 +123,7 @@ class VideoDecoder( val codec: MediaCodec, val bufferIndex: Int, val info: MediaCodec.BufferInfo, - val timestampUs: Long // 帧时间戳，单位微秒 + val timestampUs: Long, // 帧时间戳，单位微秒 ) // endregion @@ -128,6 +141,9 @@ class VideoDecoder( // 创建并配置MediaFormat val format = MediaFormat.createVideoFormat(mime, width, height) format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, width * height) + format.setInteger(MediaFormat.KEY_FRAME_RATE, renderFps) + format.setInteger(MediaFormat.KEY_LOW_LATENCY, 1); + // 创建解码器 val decoder = MediaCodec.createDecoderByType(mime) // 设置解码回调 @@ -198,63 +214,36 @@ class VideoDecoder( decoder.start() mediaCodec = decoder - // 启动渲染线程 - renderThreadRunning = true - renderThread = Thread { - var hasNotifiedFlutter = false - var renderedFrameCount = 0 // 渲染帧计数器 - val fpsAdjustInterval = 10 // 每渲染10帧调整一次帧率 - while (renderThreadRunning) { - // 计算每帧渲染间隔 - val frameIntervalMs = if (renderFps > 0) 1000L / renderFps else 66L - val loopStart = SystemClock.elapsedRealtime() - try { - // 阻塞式等待新帧，避免空转 - val frame = outputFrameQueue.take() + // 启动定时渲染任务，实现完全线性调度 + // 说明：本方案通过ScheduledExecutorService定时驱动渲染，每帧间隔严格等距，不依赖阻塞或sleep + var hasNotifiedFlutter = false + var renderedFrameCount = 0 // 渲染帧计数器 + val renderTask = Runnable { + try { + val frame = outputFrameQueue.poll() + if (frame != null) { frame.codec.releaseOutputBuffer(frame.bufferIndex, true) - // 7. 渲染线程用系统时间推进 latestRenderedTimestampMs = System.currentTimeMillis() renderedFrameCount++ - // 只在首次渲染时回调Flutter if (!hasNotifiedFlutter) { mainHandler.post { onFrameRendered() } hasNotifiedFlutter = true } - // 每渲染N帧动态调整一次帧率 - if (renderedFrameCount % fpsAdjustInterval == 0) { - val measuredFps = calculateDecodeFps() - val newFps = updateSmoothedFps(measuredFps) - renderFps = newFps - Log.i(TAG, "[AutoFps] measuredFps=$measuredFps, smoothedFps=$smoothedFps, renderFps=$renderFps") - } - } catch (e: Exception) { - Log.e(TAG, "[RenderThread] Exception", e) } - // 控制渲染节奏 - val loopCost = SystemClock.elapsedRealtime() - loopStart - val sleepMs = frameIntervalMs - loopCost - if (sleepMs > 0) { - try { Thread.sleep(sleepMs) } catch (_: Exception) {} - } else { - // 若解码极慢，sleepMs为负，直接进入下一帧，防止阻塞 - } - } - // 清理剩余帧，防止内存泄漏 - while (true) { - val frame = outputFrameQueue.poll() ?: break - try { - frame.codec.releaseOutputBuffer(frame.bufferIndex, false) - } catch (_: Exception) {} + // 若outputFrameQueue为空，跳过本次渲染，实现线性调度 + } catch (e: Exception) { + Log.e(TAG, "[RenderTask] Exception", e) } } - renderThread?.start() + // 固定20fps渲染（50ms间隔） + scheduler.scheduleAtFixedRate(renderTask, 0, 50, java.util.concurrent.TimeUnit.MILLISECONDS) } // endregion // region 核心方法 /** - * 向解码器输入一帧数据（所有类型均允许入队，去重） + * 向解码器输入一帧数据（支持I/P帧乱序，P帧依赖未满足时暂存） */ fun decodeFrame( frameData: ByteArray, @@ -280,30 +269,49 @@ class VideoDecoder( val absTimestamp = base + (timestamp - firstRel) // 3. decodeFrame延迟丢弃判断（用系统时间） val now = System.currentTimeMillis() - val diff = now - absTimestamp - // Log.d(TAG, "[decodeFrame] absTimestamp=$absTimestamp, now=$now, now-absTimestamp=$diff, maxDelay=$MAX_ALLOWED_DELAY_MS") if (absTimestamp < now - MAX_ALLOWED_DELAY_MS) { Log.w(TAG, "[decodeFrame] Drop frame due to delay: absFrameTs=$absTimestamp, now=$now, maxDelay=$MAX_ALLOWED_DELAY_MS") return false } - var allow = false - if (frameType == 0) { // I帧 - lastIFrameSeq = frameSeq - allow = true - } else { - val lastI = lastIFrameSeq - if (lastI == null) { - Log.w(TAG, "[decodeFrame] Drop P/B frame: no I-frame yet, frameSeq=$frameSeq, refIFrameSeq=$refIFrameSeq") - return false - } - allow = (refIFrameSeq == lastI) - if (!allow) { - Log.w(TAG, "[decodeFrame] Drop P/B frame: refIFrameSeq=$refIFrameSeq != lastIFrameSeq=$lastI, frameSeq=$frameSeq") - return false + // ===== 帧重排序缓冲区机制 ===== + reorderLock.withLock { + if (frameType == 0) { // I帧 + receivedIFrames.add(frameSeq) + lastIFrameSeq = frameSeq + // I帧直接入解码队列 + inputFrameQueue.offer(FrameData(frameData, frameType, timestamp, frameSeq, refIFrameSeq), 50, TimeUnit.MILLISECONDS) + // 检查缓冲区，入队所有依赖于该I帧的P帧 + val readyPFrames = reorderBuffer.values.filter { it.refIFrameSeq == frameSeq } + .sortedBy { it.frameSeq } + for (pFrame in readyPFrames) { + inputFrameQueue.offer(pFrame, 50, TimeUnit.MILLISECONDS) + reorderBuffer.remove(pFrame.frameSeq) + } + // 清理过期P帧（如缓冲区过大） + if (reorderBuffer.size > MAX_REORDER_BUFFER_SIZE) { + val toRemove = reorderBuffer.keys.sorted().take(reorderBuffer.size - MAX_REORDER_BUFFER_SIZE) + toRemove.forEach { reorderBuffer.remove(it) } + } + return true + } else { // P帧 + val lastI = lastIFrameSeq + // 只有依赖的I帧已收到，才允许入队，否则暂存 + if (refIFrameSeq != null && receivedIFrames.contains(refIFrameSeq)) { + inputFrameQueue.offer(FrameData(frameData, frameType, timestamp, frameSeq, refIFrameSeq), 50, TimeUnit.MILLISECONDS) + return true + } else { + // 暂存到重排序缓冲区 + reorderBuffer[frameSeq] = FrameData(frameData, frameType, timestamp, frameSeq, refIFrameSeq) + // 控制缓冲区大小 + if (reorderBuffer.size > MAX_REORDER_BUFFER_SIZE) { + val toRemove = reorderBuffer.keys.sorted().take(reorderBuffer.size - MAX_REORDER_BUFFER_SIZE) + toRemove.forEach { reorderBuffer.remove(it) } + } + Log.w(TAG, "[decodeFrame] P-frame cached: frameSeq=$frameSeq, refIFrameSeq=$refIFrameSeq, waiting for I-frame.") + return false + } } } - // 4. 入队时FrameData仍用原始相对时间戳 - return inputFrameQueue.offer(FrameData(frameData, frameType, timestamp, frameSeq, refIFrameSeq), 50, TimeUnit.MILLISECONDS) } /** @@ -312,10 +320,9 @@ class VideoDecoder( fun release() { running = false inputFrameQueue.clear() - // 停止渲染线程 - renderThreadRunning = false + // 停止定时渲染任务 try { - renderThread?.join(200) + scheduler.shutdownNow() } catch (_: Exception) {} outputFrameQueue.clear() try { diff --git a/lib/video_decode_plugin.dart b/lib/video_decode_plugin.dart index 675a250..c087e99 100644 --- a/lib/video_decode_plugin.dart +++ b/lib/video_decode_plugin.dart @@ -180,6 +180,8 @@ class VideoDecodePlugin { frameSeq: frameSeq - 1, refIFrameSeq: frameSeq - 1, ); + // 提取i帧 + frameData = NaluUtils.filterNalusByType(frameData, 5); await _decodeFrame( frameData: Uint8List.fromList(frameData), frameType: 0, @@ -223,6 +225,8 @@ class VideoDecodePlugin { frameSeq: frameSeq - 1, refIFrameSeq: frameSeq - 1, ); + // 提取i帧 + frameData = NaluUtils.filterNalusByType(frameData, 5); await _decodeFrame( frameData: Uint8List.fromList(frameData), frameType: 0,