From 9f97db8852433b961b8a026adbbb9de896549576 Mon Sep 17 00:00:00 2001 From: liyi Date: Wed, 23 Apr 2025 10:37:53 +0800 Subject: [PATCH] =?UTF-8?q?feat:v1=E7=89=88=E6=9C=AC=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../video_decode_plugin/VideoDecodePlugin.kt | 7 +- .../video_decode_plugin/VideoDecoder.kt | 196 ++++++- .../video_decode_plugin/VideoDecoderConfig.kt | 14 +- example/lib/main.dart | 535 ++++++++++++++++-- lib/video_decode_plugin.dart | 250 ++++++++ 5 files changed, 916 insertions(+), 86 deletions(-) diff --git a/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecodePlugin.kt b/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecodePlugin.kt index cd3e2b9..c100783 100644 --- a/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecodePlugin.kt +++ b/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecodePlugin.kt @@ -159,7 +159,12 @@ class VideoDecodePlugin : FlutterPlugin, MethodCallHandler { enableHardwareDecoder = enableHardwareDecoder, threadCount = threadCount, bufferSize = bufferSize, - isDebug = isDebug + isDebug = isDebug, + enableDynamicThresholds = call.argument("enableDynamicThresholds") ?: true, + initialMaxPFrames = call.argument("initialMaxPFrames") ?: 10, + initialIFrameTimeoutMs = call.argument("initialIFrameTimeoutMs") ?: 500, + minMaxPFrames = call.argument("minMaxPFrames") ?: 5, + maxMaxPFrames = call.argument("maxMaxPFrames") ?: 30 ) // 创建解码器 diff --git a/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoder.kt b/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoder.kt index fe1c7e9..520938d 100644 --- a/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoder.kt +++ b/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoder.kt @@ -36,7 +36,10 @@ class VideoDecoder( private const val NAL_UNIT_TYPE_NON_IDR = 1 // P帧 // 最大允许连续P帧数 - private const val MAX_CONSECUTIVE_P_FRAMES = 30 + private const val MAX_CONSECUTIVE_P_FRAMES = 10 + + // I帧超时时间（毫秒）- 超过此时间没有收到I帧则丢弃P帧 + private const val MAX_IFRAME_TIMEOUT_MS = 500 // 异步模式参数 private const val LOW_LATENCY_MODE = true @@ -79,6 +82,22 @@ class VideoDecoder( // 连续P帧计数 private var consecutivePFrameCount = 0 + // FPS统计相关变量 + private var fpsCalculationStartTime = 0L + private var framesForFps = 0 + private var currentFps = 0f + private val FPS_UPDATE_INTERVAL_MS = 1000 // 每秒更新一次FPS + + // 动态参数调整相关变量 + private var detectedGopSize = 0 + private var lastDetectedIFrameTime = 0L + private var iFrameIntervals = mutableListOf() + private val GOP_HISTORY_SIZE = 5 // 记录最近5个GOP间隔 + + // 动态阈值参数 + private var dynamicMaxConsecutivePFrames = config.initialMaxPFrames + private var dynamicIFrameTimeout = config.initialIFrameTimeoutMs + // 用于避免重复处理相同SPS/PPS的缓存 private var lastSPSHash: Int? = null private var lastPPSHash: Int? = null @@ -174,6 +193,24 @@ class VideoDecoder( return } + // FPS计算 + val currentTime = System.currentTimeMillis() + if (fpsCalculationStartTime == 0L) { + fpsCalculationStartTime = currentTime + framesForFps = 0 + } + + framesForFps++ + + // 每秒更新一次FPS + if (currentTime - fpsCalculationStartTime >= FPS_UPDATE_INTERVAL_MS) { + val elapsedSeconds = (currentTime - fpsCalculationStartTime) / 1000f + currentFps = framesForFps / elapsedSeconds + logDebug("当前渲染FPS: $currentFps") + fpsCalculationStartTime = currentTime + framesForFps = 0 + } + try { logDebug("发送帧可用通知，当前渲染帧数: $renderedFrameCount") callback?.onFrameAvailable() @@ -294,7 +331,42 @@ class VideoDecoder( hasSentPPS.set(true) } else if (effectiveType == NAL_UNIT_TYPE_IDR) { hasSentIDR.set(true) - lastIFrameTimeMs = System.currentTimeMillis() + val currentTime = System.currentTimeMillis() + + // 计算I帧间隔并更新动态参数 + if (config.enableDynamicThresholds && lastDetectedIFrameTime > 0) { + val iFrameInterval = currentTime - lastDetectedIFrameTime + + // 添加到历史记录 + iFrameIntervals.add(iFrameInterval) + if (iFrameIntervals.size > GOP_HISTORY_SIZE) { + iFrameIntervals.removeAt(0) + } + + // 计算平均GOP大小 + if (iFrameIntervals.isNotEmpty()) { + val avgIFrameInterval = iFrameIntervals.average().toLong() + val frameRate = config.frameRate ?: 30 + detectedGopSize = (avgIFrameInterval * frameRate / 1000).toInt() + + if (detectedGopSize > 0) { + // 动态调整最大连续P帧阈值 - 设置为GOP的1.5倍，但受配置限制 + val newMaxPFrames = (detectedGopSize * 1.5).toInt() + dynamicMaxConsecutivePFrames = newMaxPFrames.coerceIn( + config.minMaxPFrames, + config.maxMaxPFrames + ) + + // 动态调整I帧超时时间 - 设置为平均I帧间隔的2倍，但至少为200ms + dynamicIFrameTimeout = Math.max(200, avgIFrameInterval.toInt() * 2) + + logDebug("动态参数更新: GOP=$detectedGopSize, 最大P帧=$dynamicMaxConsecutivePFrames, I帧超时=${dynamicIFrameTimeout}ms") + } + } + } + + lastDetectedIFrameTime = currentTime + lastIFrameTimeMs = currentTime consecutivePFrameCount = 0 } else { // P帧处理 @@ -305,6 +377,41 @@ class VideoDecoder( } consecutivePFrameCount++ + + // 检查连续P帧数是否超过阈值 - 使用动态阈值或固定阈值 + val maxPFrames = if (config.enableDynamicThresholds) + dynamicMaxConsecutivePFrames + else + MAX_CONSECUTIVE_P_FRAMES + + if (consecutivePFrameCount >= maxPFrames) { + logWarning("丢弃P帧，因为连续P帧过多($consecutivePFrameCount > $maxPFrames)") + droppedFrameCount++ + return false + } + + // 检查是否自上一个I帧过去太久 - 使用动态阈值或固定阈值 + if (lastIFrameTimeMs > 0) { + val timeSinceLastIFrame = System.currentTimeMillis() - lastIFrameTimeMs + val iFrameTimeout = if (config.enableDynamicThresholds) + dynamicIFrameTimeout + else + MAX_IFRAME_TIMEOUT_MS + + if (timeSinceLastIFrame > iFrameTimeout) { + logWarning("丢弃P帧，因为距离上一个I帧时间过长(${timeSinceLastIFrame}ms > ${iFrameTimeout}ms)") + droppedFrameCount++ + return false + } + } + + // 帧大小异常检测 - 如果帧过小或过大，可能是损坏的帧 + val expectedFrameSize = config.width * config.height / 8 // 粗略估计 + if (frameData.size < 10 || frameData.size > expectedFrameSize * 2) { + logWarning("丢弃帧，因为帧大小异常(${frameData.size}字节)") + droppedFrameCount++ + return false + } } // 记录帧信息 @@ -356,31 +463,62 @@ class VideoDecoder( val bufferInfo = MediaCodec.BufferInfo() var outputDone = false - while (!outputDone) { - val outputBufferIndex = codec.dequeueOutputBuffer(bufferInfo, 0) // 不等待，只检查当前可用的 - - when { - outputBufferIndex >= 0 -> { - val render = bufferInfo.size > 0 - codec.releaseOutputBuffer(outputBufferIndex, render) - - if (render) { - renderedFrameCount++ - lastOutputTimeMs = System.currentTimeMillis() - logDebug("成功渲染帧 #$renderedFrameCount") + var errorDetected = false + + while (!outputDone && !errorDetected) { + try { + val outputBufferIndex = codec.dequeueOutputBuffer(bufferInfo, 0) // 不等待，只检查当前可用的 + + when { + outputBufferIndex >= 0 -> { + val render = bufferInfo.size > 0 && (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0 - // 通知Flutter刷新纹理 - notifyFrameAvailable() + // 检查解码错误 + if ((bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + logWarning("收到结束流标志") + // 忽略错误，继续解码 + } + + if (render) { + // 释放并渲染 + codec.releaseOutputBuffer(outputBufferIndex, true) + renderedFrameCount++ + lastOutputTimeMs = System.currentTimeMillis() + logDebug("成功渲染帧 #$renderedFrameCount") + + // 通知Flutter刷新纹理 + notifyFrameAvailable() + } else { + // 释放但不渲染 + codec.releaseOutputBuffer(outputBufferIndex, false) + } + } + outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + logDebug("输出格式变更: ${codec.outputFormat}") + } + outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER -> { + outputDone = true + } + outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED -> { + logDebug("输出缓冲区已更改") + } + else -> { + // 其他错误情况 + logWarning("未知的输出缓冲区索引: $outputBufferIndex") + errorDetected = true } } - outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { - logDebug("输出格式变更: ${codec.outputFormat}") - } - outputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER -> { - outputDone = true - } + } catch (e: Exception) { + logError("处理输出缓冲区时出错", e) + errorDetected = true } } + + // 如果检测到错误，可以考虑重置编码器状态或清除缓冲区 + if (errorDetected) { + logWarning("检测到解码错误，等待下一个关键帧...") + // 在实际应用中，可以考虑在这里执行更复杂的恢复逻辑 + } } /** @@ -431,13 +569,25 @@ class VideoDecoder( "totalFrames" to frameCount, "renderedFrames" to renderedFrameCount, "droppedFrames" to droppedFrameCount, + "fps" to currentFps, + "detectedGopSize" to detectedGopSize, + "dynamicMaxConsecutivePFrames" to dynamicMaxConsecutivePFrames, + "dynamicIFrameTimeoutMs" to dynamicIFrameTimeout, "hasSentSPS" to hasSentSPS.get(), "hasSentPPS" to hasSentPPS.get(), "hasSentIDR" to hasSentIDR.get(), "consecutivePFrames" to consecutivePFrameCount, "targetWidth" to config.width, "targetHeight" to config.height, - "frameRate" to (config.frameRate ?: 0) + "frameRate" to (config.frameRate ?: 0), + "enableDynamicThresholds" to config.enableDynamicThresholds ) } + + /** + * 获取当前渲染FPS + */ + fun getCurrentFps(): Float { + return currentFps + } } \ No newline at end of file diff --git a/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoderConfig.kt b/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoderConfig.kt index 6efcbdc..ceb3091 100644 --- a/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoderConfig.kt +++ b/android/src/main/kotlin/top/skychip/video_decode_plugin/VideoDecoderConfig.kt @@ -11,6 +11,11 @@ package top.skychip.video_decode_plugin * @param threadCount 解码线程数 * @param bufferSize 输入缓冲区大小 * @param isDebug 是否开启调试日志 + * @param enableDynamicThresholds 是否启用动态阈值 + * @param initialMaxPFrames 初始最大连续P帧数 + * @param initialIFrameTimeoutMs 初始I帧超时时间 + * @param minMaxPFrames 最小最大连续P帧数 + * @param maxMaxPFrames 最大最大连续P帧数 */ data class VideoDecoderConfig( val width: Int, @@ -19,6 +24,11 @@ data class VideoDecoderConfig( val frameRate: Int? = null, val enableHardwareDecoder: Boolean = true, val threadCount: Int = 1, - val bufferSize: Int = 10, - val isDebug: Boolean = false + val bufferSize: Int = 30, + val isDebug: Boolean = false, + val enableDynamicThresholds: Boolean = true, + val initialMaxPFrames: Int = 10, + val initialIFrameTimeoutMs: Int = 500, + val minMaxPFrames: Int = 5, + val maxMaxPFrames: Int = 30 ) \ No newline at end of file diff --git a/example/lib/main.dart b/example/lib/main.dart index 9ecdb7b..95d54b9 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -68,9 +68,40 @@ class H264Frame { class NalUnitType { static const int UNSPECIFIED = 0; static const int CODED_SLICE_NON_IDR = 1; // P帧 + static const int PARTITION_A = 2; + static const int PARTITION_B = 3; + static const int PARTITION_C = 4; static const int CODED_SLICE_IDR = 5; // I帧 + static const int SEI = 6; // 补充增强信息 static const int SPS = 7; // 序列参数集 static const int PPS = 8; // 图像参数集 + static const int AUD = 9; // 访问单元分隔符 + static const int END_SEQUENCE = 10; // 序列结束 + static const int END_STREAM = 11; // 码流结束 + static const int FILLER_DATA = 12; // 填充数据 + static const int SPS_EXT = 13; // SPS扩展 + static const int PREFIX_NAL = 14; // 前缀NAL单元 + static const int SUBSET_SPS = 15; // 子集SPS + static const int DEPTH_PARAM_SET = 16; // 深度参数集 + static const int RESERVED_17 = 17; // 保留类型 + static const int RESERVED_18 = 18; // 保留类型 + static const int CODED_SLICE_AUX = 19; // 辅助切片 + static const int CODED_SLICE_EXTENSION = 20; // 扩展切片 + static const int CODED_SLICE_DEPTH = 21; // 深度扩展 + static const int RESERVED_22 = 22; // 保留类型 + static const int RESERVED_23 = 23; // 保留类型 + static const int STAP_A = 24; // 单时间聚合包A + static const int STAP_B = 25; // 单时间聚合包B + static const int MTAP_16 = 26; // 多时间聚合包16 + static const int MTAP_24 = 27; // 多时间聚合包24 + static const int FU_A = 28; // 分片单元A + static const int FU_B = 29; // 分片单元B + static const int RESERVED_30 = 30; // 保留类型 + static const int RESERVED_31 = 31; // 保留类型 + + // 帧类型别名，方便调用 + static const int NON_IDR_PICTURE = CODED_SLICE_NON_IDR; // P帧 + static const int IDR_PICTURE = CODED_SLICE_IDR; // I帧 // 获取类型名称 static String getName(int type) { @@ -79,12 +110,42 @@ class NalUnitType { return "未指定"; case CODED_SLICE_NON_IDR: return "P帧"; + case PARTITION_A: + return "分区A"; + case PARTITION_B: + return "分区B"; + case PARTITION_C: + return "分区C"; case CODED_SLICE_IDR: return "I帧"; + case SEI: + return "SEI"; case SPS: return "SPS"; case PPS: return "PPS"; + case AUD: + return "AUD"; + case END_SEQUENCE: + return "序列结束"; + case END_STREAM: + return "码流结束"; + case FILLER_DATA: + return "填充数据"; + case SPS_EXT: + return "SPS扩展"; + case PREFIX_NAL: + return "前缀NAL"; + case SUBSET_SPS: + return "子集SPS"; + case CODED_SLICE_AUX: + return "辅助切片"; + case CODED_SLICE_EXTENSION: + return "扩展切片"; + case FU_A: + return "分片单元A"; + case FU_B: + return "分片单元B"; default: return "未知($type)"; } @@ -130,6 +191,26 @@ class _VideoViewState extends State { int _renderedFrameCount = 0; DateTime? _lastFrameTime; double _fps = 0; + double _decoderFps = 0; // 解码器内部计算的FPS + + // 动态阈值参数 + int _detectedGopSize = 0; + int _dynamicMaxPFrames = 0; + int _dynamicIFrameTimeoutMs = 0; + bool _enableDynamicThresholds = true; + + // 用于刷新解码器统计信息的定时器 + Timer? _statsTimer; + + // 丢包模拟相关 + bool _enablePacketLoss = false; + double _packetLossRate = 0.1; // 默认10%丢包率 + bool _dropIFrames = false; + bool _dropPFrames = false; + bool _dropSPSPPS = false; + bool _burstPacketLossMode = false; + int _burstPacketLossCounter = 0; + int _droppedFramesCount = 0; // H264文件解析 Uint8List? _h264FileData; @@ -143,10 +224,21 @@ class _VideoViewState extends State { final List _logs = []; final ScrollController _logScrollController = ScrollController(); + // 视频显示相关的属性 + bool _showingErrorFrame = false; + Timer? _errorFrameResetTimer; + @override void initState() { super.initState(); _loadH264File(); + + // 启动定时器刷新解码器统计信息 + _statsTimer = Timer.periodic(Duration(milliseconds: 1000), (timer) { + if (_isInitialized && _textureId != null) { + _updateDecoderStats(); + } + }); } @override @@ -154,6 +246,7 @@ class _VideoViewState extends State { _stopPlaying(); _releaseDecoder(); _frameTimer?.cancel(); + _statsTimer?.cancel(); // 停止统计信息更新定时器 super.dispose(); } @@ -270,62 +363,35 @@ class _VideoViewState extends State { return -1; } - // 查找NAL类型的辅助方法（用于调试） + // 获取NAL类型 + // 获取NAL类型 int _getNalType(Uint8List data) { - // 打印头几个字节 - String headerBytes = ''; - for (int i = 0; i < math.min(16, data.length); i++) { - headerBytes += '${data[i].toRadixString(16).padLeft(2, '0')} '; - } - _log("帧数据头: $headerBytes"); - - // 尝试找到起始码位置 - int nalOffset = -1; - - // 检查标准起始码 - if (data.length > 4 && - data[0] == 0x00 && - data[1] == 0x00 && - data[2] == 0x00 && - data[3] == 0x01) { - nalOffset = 4; - _log("找到4字节起始码 (0x00000001) 位置: 0"); - } else if (data.length > 3 && - data[0] == 0x00 && - data[1] == 0x00 && - data[2] == 0x01) { - nalOffset = 3; - _log("找到3字节起始码 (0x000001) 位置: 0"); - } else { - // 尝试搜索起始码 - for (int i = 0; i < data.length - 4; i++) { - if (data[i] == 0x00 && - data[i + 1] == 0x00 && - data[i + 2] == 0x00 && - data[i + 3] == 0x01) { - nalOffset = i + 4; - _log("在偏移量 $i 处找到4字节起始码"); - break; - } else if (i < data.length - 3 && - data[i] == 0x00 && - data[i + 1] == 0x00 && - data[i + 2] == 0x01) { - nalOffset = i + 3; - _log("在偏移量 $i 处找到3字节起始码"); - break; + // 跳过起始码后再获取NAL单元类型 + if (data.length > 4) { + // 检查是否有0x00000001的起始码 + if (data[0] == 0x00 && + data[1] == 0x00 && + data[2] == 0x00 && + data[3] == 0x01) { + // 起始码后的第一个字节的低5位是NAL类型 + if (data.length > 4) { + return data[4] & 0x1F; + } + } + // 检查是否有0x000001的起始码 + else if (data[0] == 0x00 && data[1] == 0x00 && data[2] == 0x01) { + // 起始码后的第一个字节的低5位是NAL类型 + if (data.length > 3) { + return data[3] & 0x1F; } } } - // 如果找到了起始码 - if (nalOffset >= 0 && nalOffset < data.length) { - int nalType = data[nalOffset] & 0x1F; - _log("解析NAL类型: ${NalUnitType.getName(nalType)} ($nalType)"); - return nalType; + // 如果找不到起始码或数据不足，则尝试直接获取 + if (data.length > 0) { + return data[0] & 0x1F; } - - _log("无法解析NAL类型"); - return -1; + return 0; } // 当新帧可用时调用 @@ -358,6 +424,11 @@ class _VideoViewState extends State { frameRate: 30, bufferSize: 30, isDebug: true, // 打开调试日志 + enableDynamicThresholds: _enableDynamicThresholds, // 使用动态阈值 + initialMaxPFrames: 60, // 初始最大连续P帧数 + initialIFrameTimeoutMs: 5000, // 初始I帧超时时间 + minMaxPFrames: 5, // 最小最大连续P帧数 + maxMaxPFrames: 60, // 最大最大连续P帧数 ); final textureId = await VideoDecodePlugin.initDecoder(config); @@ -540,18 +611,83 @@ class _VideoViewState extends State { } final frame = _h264Frames[_currentFrameIndex]; - await _decodeNextFrame(frame); + bool decodeSuccess = await _decodeNextFrame(frame); + + // 只有在成功解码的情况下才显示日志信息 + if (!decodeSuccess && _enablePacketLoss) { + _log("跳过索引 $_currentFrameIndex 的帧（丢帧模拟）"); + } + + // 无论解码是否成功，都移动到下一帧 _currentFrameIndex++; }); } - Future _decodeNextFrame(H264Frame frame) async { - if (_textureId == null || !_isInitialized) return; + Future _decodeNextFrame(H264Frame frame) async { + if (_textureId == null || !_isInitialized || !_isPlaying) { + return false; + } try { - // 检查帧的NAL类型（仅用于调试） + // 获取NAL类型 int nalType = _getNalType(frame.data); + // 模拟丢包 + if (_enablePacketLoss) { + bool shouldDrop = false; + + // 爆发式丢包模式 + if (_burstPacketLossMode && _burstPacketLossCounter > 0) { + shouldDrop = true; + _burstPacketLossCounter--; + } + // 随机丢包 + else if (math.Random().nextDouble() < _packetLossRate) { + shouldDrop = true; + + // 触发爆发式丢包 + if (_burstPacketLossMode) { + _burstPacketLossCounter = math.Random().nextInt(5) + 1; // 随机爆发1-5个包 + } + } + + // 特定类型NAL的丢包策略 + if (nalType == NalUnitType.CODED_SLICE_IDR && _dropIFrames) { + shouldDrop = true; + } else if ((nalType == NalUnitType.CODED_SLICE_NON_IDR || + nalType == NalUnitType.CODED_SLICE_EXTENSION) && + _dropPFrames) { + shouldDrop = true; + } else if ((nalType == NalUnitType.SPS || nalType == NalUnitType.PPS) && + _dropSPSPPS) { + shouldDrop = true; + } + + if (shouldDrop) { + _droppedFramesCount++; + String nalTypeName = NalUnitType.getName(nalType); + _log("丢弃帧：NAL类型 = $nalTypeName"); + + // 显示丢帧效果 + setState(() { + _showingErrorFrame = true; + }); + + // 1秒后重置丢帧效果指示器 + _errorFrameResetTimer?.cancel(); + _errorFrameResetTimer = Timer(Duration(milliseconds: 1000), () { + if (mounted) { + setState(() { + _showingErrorFrame = false; + }); + } + }); + + return false; // 直接返回false，不进行解码 + } + } + + // 解码帧 final success = await VideoDecodePlugin.decodeFrameForTexture( _textureId!, frame.data, @@ -559,14 +695,16 @@ class _VideoViewState extends State { ); if (!success) { - _log( - "解码帧失败，索引 $_currentFrameIndex (${frame.type}), NAL类型: ${NalUnitType.getName(nalType)}"); + _log("解码帧失败，索引 $_currentFrameIndex (${frame.type})"); } else { + String nalTypeName = NalUnitType.getName(nalType); _log( - "解码帧成功，索引 $_currentFrameIndex (${frame.type}), NAL类型: ${NalUnitType.getName(nalType)}"); + "解码帧成功，索引 $_currentFrameIndex (${frame.type}), NAL类型: $nalTypeName"); } + return success; } catch (e) { _log("解码帧错误: $e"); + return false; } } @@ -632,6 +770,18 @@ class _VideoViewState extends State { ), ), + // 丢包效果指示器 - 当有帧被丢弃时，上方显示红色条带 + if (_enablePacketLoss && _droppedFramesCount > 0) + Positioned( + top: 0, + left: 0, + right: 0, + height: 5, + child: Container( + color: Colors.red.withOpacity(0.8), + ), + ), + // 显示帧计数 - 调试用 Positioned( right: 10, @@ -640,7 +790,7 @@ class _VideoViewState extends State { padding: EdgeInsets.all(5), color: Colors.black.withOpacity(0.5), child: Text( - '帧: $_renderedFrameCount', + '帧: $_renderedFrameCount${_enablePacketLoss ? ' (丢帧: $_droppedFramesCount)' : ''}', style: TextStyle(color: Colors.white, fontSize: 12), ), ), @@ -702,7 +852,18 @@ class _VideoViewState extends State { Text('状态: $_statusText', style: TextStyle( fontWeight: FontWeight.bold)), - Text('FPS: ${_fps.toStringAsFixed(1)}'), + // Text('计算FPS: ${_fps.toStringAsFixed(1)}'), + ], + ), + Row( + mainAxisAlignment: + MainAxisAlignment.spaceBetween, + children: [ + Text( + '解码器FPS: ${_decoderFps.toStringAsFixed(1)}',style: TextStyle( + color: Colors.green + ),), + Text('已渲染帧数: $_renderedFrameCount'), ], ), if (_error.isNotEmpty) @@ -714,12 +875,30 @@ class _VideoViewState extends State { mainAxisAlignment: MainAxisAlignment.spaceBetween, children: [ - Text('已渲染帧数: $_renderedFrameCount'), + Text('检测到的GOP: $_detectedGopSize'), Text('解析的帧数: ${_h264Frames.length}'), ], ), Text( 'H264文件大小: ${(_h264FileData?.length ?? 0) / 1024} KB'), + + // 动态阈值参数显示 + if (_enableDynamicThresholds) + Padding( + padding: const EdgeInsets.only(top: 4.0), + child: Column( + crossAxisAlignment: + CrossAxisAlignment.start, + children: [ + Text('动态阈值参数:', + style: TextStyle( + fontWeight: FontWeight.bold)), + Text('最大连续P帧: $_dynamicMaxPFrames'), + Text( + 'I帧超时: ${_dynamicIFrameTimeoutMs}ms'), + ], + ), + ), ], ), ), @@ -770,6 +949,175 @@ class _VideoViewState extends State { ), ), + // 丢包模拟控制面板 + Card( + child: Padding( + padding: const EdgeInsets.all(8.0), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + mainAxisAlignment: + MainAxisAlignment.spaceBetween, + children: [ + Text('丢包模拟', + style: TextStyle( + fontWeight: FontWeight.bold, + fontSize: 16)), + Switch( + value: _enablePacketLoss, + onChanged: (value) { + setState(() { + _enablePacketLoss = value; + if (value) { + _droppedFramesCount = 0; // 重置丢帧计数 + } + }); + }, + ), + ], + ), + + // 动态阈值开关 + Row( + mainAxisAlignment: + MainAxisAlignment.spaceBetween, + children: [ + Text('动态阈值', + style: TextStyle( + fontWeight: FontWeight.bold)), + Switch( + value: _enableDynamicThresholds, + onChanged: (value) { + setState(() { + _enableDynamicThresholds = value; + // 需要重新初始化解码器以应用新设置 + if (_isInitialized) { + _log("更改动态阈值设置，需要重新初始化解码器"); + // 如果正在播放，先停止 + if (_isPlaying) { + _stopPlaying(); + } + // 延迟一下再重新初始化 + Future.delayed( + Duration(milliseconds: 100), () { + _initializeDecoder(); + }); + } + }); + }, + ), + ], + ), + + Divider(), + + // 丢包率控制 + Row( + children: [ + Text( + '丢包率: ${(_packetLossRate * 100).toStringAsFixed(0)}%'), + Expanded( + child: Slider( + value: _packetLossRate, + min: 0.0, + max: 1.0, + divisions: 20, + onChanged: _enablePacketLoss + ? (value) { + setState(() { + _packetLossRate = value; + }); + } + : null, + ), + ), + ], + ), + + // 丢包模式选择 + Row( + children: [ + Expanded( + child: CheckboxListTile( + dense: true, + title: Text('爆发式丢包'), + value: _burstPacketLossMode, + onChanged: _enablePacketLoss + ? (value) { + setState(() { + _burstPacketLossMode = value!; + }); + } + : null, + ), + ), + Expanded( + child: CheckboxListTile( + dense: true, + title: Text('丢弃I帧'), + value: _dropIFrames, + onChanged: _enablePacketLoss + ? (value) { + setState(() { + _dropIFrames = value!; + }); + } + : null, + ), + ), + ], + ), + + Row( + children: [ + Expanded( + child: CheckboxListTile( + dense: true, + title: Text('丢弃P帧'), + value: _dropPFrames, + onChanged: _enablePacketLoss + ? (value) { + setState(() { + _dropPFrames = value!; + }); + } + : null, + ), + ), + Expanded( + child: CheckboxListTile( + dense: true, + title: Text('丢弃SPS/PPS'), + value: _dropSPSPPS, + onChanged: _enablePacketLoss + ? (value) { + setState(() { + _dropSPSPPS = value!; + }); + } + : null, + ), + ), + ], + ), + + // 丢包统计信息 + Padding( + padding: const EdgeInsets.only(top: 8.0), + child: Text( + '已丢弃帧数: $_droppedFramesCount', + style: TextStyle( + color: _droppedFramesCount > 0 + ? Colors.red + : Colors.black), + ), + ), + ], + ), + ), + ), + // 日志区域 SizedBox(height: 8), Text('日志:', @@ -810,4 +1158,71 @@ class _VideoViewState extends State { ), ); } + + // 更新解码器统计信息 + Future _updateDecoderStats() async { + if (_textureId == null || !_isInitialized) return; + + try { + // 获取FPS + final fps = await VideoDecodePlugin.getCurrentFps(_textureId); + + // 获取动态阈值参数 + final thresholdParams = + await VideoDecodePlugin.getDynamicThresholdParams(_textureId); + + if (mounted) { + setState(() { + _decoderFps = fps; + _detectedGopSize = thresholdParams['detectedGopSize'] ?? 0; + _dynamicMaxPFrames = + thresholdParams['dynamicMaxConsecutivePFrames'] ?? 0; + _dynamicIFrameTimeoutMs = + thresholdParams['dynamicIFrameTimeoutMs'] ?? 0; + _enableDynamicThresholds = + thresholdParams['enableDynamicThresholds'] ?? true; + }); + } + } catch (e) { + _log("获取解码器统计信息失败: $e"); + } + } +} + +// 添加错误帧绘制器 +class ErrorFramePainter extends CustomPainter { + @override + void paint(Canvas canvas, Size size) { + final random = math.Random(); + final paint = Paint() + ..color = Colors.red.withOpacity(0.2) + ..style = PaintingStyle.fill; + + // 绘制随机条纹效果 + for (int i = 0; i < 20; i++) { + double y = random.nextDouble() * size.height; + canvas.drawRect( + Rect.fromLTWH(0, y, size.width, 3 + random.nextDouble() * 5), + paint, + ); + } + + // 绘制马赛克块 + for (int i = 0; i < 50; i++) { + double x = random.nextDouble() * size.width; + double y = random.nextDouble() * size.height; + double blockSize = 10 + random.nextDouble() * 30; + + canvas.drawRect( + Rect.fromLTWH(x, y, blockSize, blockSize), + Paint() + ..color = Color.fromRGBO(random.nextInt(255), random.nextInt(255), + random.nextInt(255), 0.3)); + } + } + + @override + bool shouldRepaint(covariant CustomPainter oldDelegate) { + return true; // 每次都重新绘制以产生动态效果 + } } diff --git a/lib/video_decode_plugin.dart b/lib/video_decode_plugin.dart index f43ec89..6aef504 100644 --- a/lib/video_decode_plugin.dart +++ b/lib/video_decode_plugin.dart @@ -7,6 +7,181 @@ import 'package:flutter/services.dart'; import 'video_decode_plugin_platform_interface.dart'; +/// H.265/HEVC NAL单元类型定义 +class HevcNalUnitType { + static const int TRAIL_N = 0; // 尾随图片 - 非参考图片 + static const int TRAIL_R = 1; // 尾随图片 - 参考图片 + + static const int TSA_N = 2; // 时间子层访问 - 非参考图片 + static const int TSA_R = 3; // 时间子层访问 - 参考图片 + + static const int STSA_N = 4; // 分步时间子层访问 - 非参考图片 + static const int STSA_R = 5; // 分步时间子层访问 - 参考图片 + + static const int RADL_N = 6; // 随机访问解码先导 - 非参考图片 + static const int RADL_R = 7; // 随机访问解码先导 - 参考图片 + + static const int RASL_N = 8; // 随机访问跳过先导 - 非参考图片 + static const int RASL_R = 9; // 随机访问跳过先导 - 参考图片 + + static const int RSV_VCL_N10 = 10; // 保留的非IRAP VCL NAL单元类型 + static const int RSV_VCL_R11 = 11; // 保留的非IRAP VCL NAL单元类型 + static const int RSV_VCL_N12 = 12; // 保留的非IRAP VCL NAL单元类型 + static const int RSV_VCL_R13 = 13; // 保留的非IRAP VCL NAL单元类型 + static const int RSV_VCL_N14 = 14; // 保留的非IRAP VCL NAL单元类型 + static const int RSV_VCL_R15 = 15; // 保留的非IRAP VCL NAL单元类型 + + static const int BLA_W_LP = 16; // 有前导的无损拼接访问 + static const int BLA_W_RADL = 17; // 有RADL的无损拼接访问 + static const int BLA_N_LP = 18; // 无前导的无损拼接访问 + + static const int IDR_W_RADL = 19; // 有RADL的瞬时解码刷新 (IDR) + static const int IDR_N_LP = 20; // 无前导的瞬时解码刷新 (IDR) + + static const int CRA_NUT = 21; // 清理随机访问 + + static const int RSV_IRAP_VCL22 = 22; // 保留的IRAP VCL NAL单元类型 + static const int RSV_IRAP_VCL23 = 23; // 保留的IRAP VCL NAL单元类型 + + static const int RSV_VCL24 = 24; // 保留的VCL NAL单元类型 + static const int RSV_VCL25 = 25; // 保留的VCL NAL单元类型 + static const int RSV_VCL26 = 26; // 保留的VCL NAL单元类型 + static const int RSV_VCL27 = 27; // 保留的VCL NAL单元类型 + static const int RSV_VCL28 = 28; // 保留的VCL NAL单元类型 + static const int RSV_VCL29 = 29; // 保留的VCL NAL单元类型 + static const int RSV_VCL30 = 30; // 保留的VCL NAL单元类型 + static const int RSV_VCL31 = 31; // 保留的VCL NAL单元类型 + + // 非VCL NAL单元类型 + static const int VPS = 32; // 视频参数集 + static const int SPS = 33; // 序列参数集 + static const int PPS = 34; // 图像参数集 + static const int AUD = 35; // 访问单元分隔符 + static const int EOS = 36; // 序列结束 + static const int EOB = 37; // 比特流结束 + static const int FD = 38; // 填充数据 + + static const int PREFIX_SEI = 39; // 前缀辅助增强信息 + static const int SUFFIX_SEI = 40; // 后缀辅助增强信息 + + static const int RSV_NVCL41 = 41; // 保留的非VCL NAL单元类型 + static const int RSV_NVCL42 = 42; // 保留的非VCL NAL单元类型 + static const int RSV_NVCL43 = 43; // 保留的非VCL NAL单元类型 + static const int RSV_NVCL44 = 44; // 保留的非VCL NAL单元类型 + static const int RSV_NVCL45 = 45; // 保留的非VCL NAL单元类型 + static const int RSV_NVCL46 = 46; // 保留的非VCL NAL单元类型 + static const int RSV_NVCL47 = 47; // 保留的非VCL NAL单元类型 + + static const int UNSPEC48 = 48; // 未指定的类型 + static const int UNSPEC49 = 49; // 未指定的类型 + static const int UNSPEC50 = 50; // 未指定的类型 + static const int UNSPEC51 = 51; // 未指定的类型 + static const int UNSPEC52 = 52; // 未指定的类型 + static const int UNSPEC53 = 53; // 未指定的类型 + static const int UNSPEC54 = 54; // 未指定的类型 + static const int UNSPEC55 = 55; // 未指定的类型 + static const int UNSPEC56 = 56; // 未指定的类型 + static const int UNSPEC57 = 57; // 未指定的类型 + static const int UNSPEC58 = 58; // 未指定的类型 + static const int UNSPEC59 = 59; // 未指定的类型 + static const int UNSPEC60 = 60; // 未指定的类型 + static const int UNSPEC61 = 61; // 未指定的类型 + static const int UNSPEC62 = 62; // 未指定的类型 + static const int UNSPEC63 = 63; // 未指定的类型 + + // 帧类型别名，方便判断 + // I帧类型：IDR_W_RADL, IDR_N_LP, BLA_W_LP, BLA_W_RADL, BLA_N_LP, CRA_NUT + static const List KEY_FRAMES = [ + IDR_W_RADL, + IDR_N_LP, + BLA_W_LP, + BLA_W_RADL, + BLA_N_LP, + CRA_NUT + ]; + + // 参数集类型：VPS, SPS, PPS + static const List PARAMETER_SETS = [VPS, SPS, PPS]; + + /// 判断是否为关键帧NAL类型 + static bool isKeyFrame(int nalUnitType) { + return KEY_FRAMES.contains(nalUnitType); + } + + /// 判断是否为参数集NAL类型 + static bool isParameterSet(int nalUnitType) { + return PARAMETER_SETS.contains(nalUnitType); + } + + /// 判断是否为IDR帧 + static bool isIdrFrame(int nalUnitType) { + return nalUnitType == IDR_W_RADL || nalUnitType == IDR_N_LP; + } + + /// 获取NAL单元类型名称 + static String getName(int type) { + switch (type) { + case TRAIL_N: + return "TRAIL_N"; + case TRAIL_R: + return "TRAIL_R"; + case TSA_N: + return "TSA_N"; + case TSA_R: + return "TSA_R"; + case STSA_N: + return "STSA_N"; + case STSA_R: + return "STSA_R"; + case RADL_N: + return "RADL_N"; + case RADL_R: + return "RADL_R"; + case RASL_N: + return "RASL_N"; + case RASL_R: + return "RASL_R"; + case BLA_W_LP: + return "BLA_W_LP"; + case BLA_W_RADL: + return "BLA_W_RADL"; + case BLA_N_LP: + return "BLA_N_LP"; + case IDR_W_RADL: + return "IDR_W_RADL"; + case IDR_N_LP: + return "IDR_N_LP"; + case CRA_NUT: + return "CRA_NUT"; + case VPS: + return "VPS"; + case SPS: + return "SPS"; + case PPS: + return "PPS"; + case AUD: + return "AUD"; + case EOS: + return "EOS"; + case EOB: + return "EOB"; + case FD: + return "FD"; + case PREFIX_SEI: + return "PREFIX_SEI"; + case SUFFIX_SEI: + return "SUFFIX_SEI"; + default: + if (type >= 10 && type <= 15) return "RSV_VCL_${type}"; + if (type >= 22 && type <= 23) return "RSV_IRAP_VCL${type}"; + if (type >= 24 && type <= 31) return "RSV_VCL${type}"; + if (type >= 41 && type <= 47) return "RSV_NVCL${type}"; + if (type >= 48 && type <= 63) return "UNSPEC${type}"; + return "未知(${type})"; + } + } +} + /// 视频帧类型 enum FrameType { /// I帧 @@ -62,6 +237,21 @@ class VideoDecoderConfig { /// 是否启用硬件解码，默认true final bool enableHardwareDecoder; + /// 是否启用动态阈值，默认true + final bool enableDynamicThresholds; + + /// 初始最大连续P帧数，默认10 + final int initialMaxPFrames; + + /// 初始I帧超时时间(毫秒)，默认500 + final int initialIFrameTimeoutMs; + + /// 最小最大连续P帧数，默认5 + final int minMaxPFrames; + + /// 最大最大连续P帧数，默认30 + final int maxMaxPFrames; + /// 构造函数 VideoDecoderConfig({ this.width = 640, @@ -72,6 +262,11 @@ class VideoDecoderConfig { this.threadCount = 1, this.isDebug = false, this.enableHardwareDecoder = true, + this.enableDynamicThresholds = true, + this.initialMaxPFrames = 10, + this.initialIFrameTimeoutMs = 500, + this.minMaxPFrames = 5, + this.maxMaxPFrames = 30, }); /// 转换为Map @@ -85,6 +280,11 @@ class VideoDecoderConfig { 'threadCount': threadCount, 'isDebug': isDebug, 'enableHardwareDecoder': enableHardwareDecoder, + 'enableDynamicThresholds': enableDynamicThresholds, + 'initialMaxPFrames': initialMaxPFrames, + 'initialIFrameTimeoutMs': initialIFrameTimeoutMs, + 'minMaxPFrames': minMaxPFrames, + 'maxMaxPFrames': maxMaxPFrames, }; } } @@ -552,6 +752,52 @@ class VideoDecodePlugin { return {}; } } + + /// 获取当前渲染FPS + /// + /// 返回当前解码器的实时渲染帧率 + /// 如果解码器未初始化或获取失败，返回0.0 + static Future getCurrentFps([int? textureId]) async { + final targetTextureId = textureId ?? _defaultTextureId; + if (targetTextureId == null) { + return 0.0; + } + + try { + final stats = await getDecoderStats(targetTextureId); + return stats['fps'] as double? ?? 0.0; + } catch (e) { + _logError('获取FPS失败: $e'); + return 0.0; + } + } + + /// 获取动态阈值参数 + /// + /// 返回当前解码器使用的动态阈值参数 + /// 包括检测到的GOP大小、最大连续P帧数限制、I帧超时时间等 + static Future> getDynamicThresholdParams( + [int? textureId]) async { + final targetTextureId = textureId ?? _defaultTextureId; + if (targetTextureId == null) { + return {}; + } + + try { + final stats = await getDecoderStats(targetTextureId); + return { + 'detectedGopSize': stats['detectedGopSize'] as int? ?? 0, + 'dynamicMaxConsecutivePFrames': + stats['dynamicMaxConsecutivePFrames'] as int? ?? 0, + 'dynamicIFrameTimeoutMs': stats['dynamicIFrameTimeoutMs'] as int? ?? 0, + 'enableDynamicThresholds': + stats['enableDynamicThresholds'] as bool? ?? false, + }; + } catch (e) { + _logError('获取动态阈值参数失败: $e'); + return {}; + } + } } /// 在Dart中实现简单的同步锁 @@ -565,3 +811,7 @@ void synchronized(Object lock, Function() action) { T synchronizedWithResult(Object lock, T Function() action) { return action(); } + + + +