From e806987fa0ae4f3b58b6699adfaf279330217d57 Mon Sep 17 00:00:00 2001 From: liyi Date: Wed, 30 Apr 2025 10:25:19 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E5=AE=8C=E6=88=90720P=2020=E5=B8=A7?= =?UTF-8?q?=E6=B8=B2=E6=9F=93=E9=9C=80=E6=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../starChart/proto/talk_expect.pbenum.dart | 2 + .../native/talk_view_native_decode_logic.dart | 921 +++++++++++++----- .../native/talk_view_native_decode_page.dart | 194 ++-- .../native/talk_view_native_decode_state.dart | 15 + 4 files changed, 774 insertions(+), 358 deletions(-) diff --git a/lib/talk/starChart/proto/talk_expect.pbenum.dart b/lib/talk/starChart/proto/talk_expect.pbenum.dart index d6b34250..10306109 100644 --- a/lib/talk/starChart/proto/talk_expect.pbenum.dart +++ b/lib/talk/starChart/proto/talk_expect.pbenum.dart @@ -19,12 +19,14 @@ class VideoTypeE extends $pb.ProtobufEnum { static const VideoTypeE H264 = VideoTypeE._(1, _omitEnumNames ? '' : 'H264'); static const VideoTypeE IMAGE = VideoTypeE._(2, _omitEnumNames ? '' : 'IMAGE'); static const VideoTypeE VP8 = VideoTypeE._(3, _omitEnumNames ? '' : 'VP8'); + static const VideoTypeE H264_720P = VideoTypeE._(4, _omitEnumNames ? '' : 'H264_720P'); static const $core.List values = [ NONE_V, H264, IMAGE, VP8, + H264_720P, ]; static final $core.Map<$core.int, VideoTypeE> _byValue = $pb.ProtobufEnum.initByValue(values); diff --git a/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart b/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart index 2fad25a7..8129f204 100644 --- a/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart +++ b/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart @@ -3,6 +3,7 @@ import 'dart:io'; import 'dart:ui' as ui; import 'dart:math'; // Import the math package to use sqrt +import 'package:flutter/cupertino.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter/services.dart'; @@ -62,41 +63,76 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { // 定义一个变量来保存上一帧的时间戳 int? _previousFrameTimestamp; + int _flutterToNativeFrameCount = 0; + int _lastFlutterToNativePrintTime = 0; + int _networkFrameCount = 0; + int _lastNetworkPrintTime = 0; + Timer? _frameRefreshTimer; + + bool _isFrameAvailable = true; + int _renderedFrameCount = 0; + int _lastRenderedFrameTime = 0; + + // 写入前的缓存队列（I帧前） + final List> _preIFrameCache = []; + bool _hasWrittenFirstIFrame = false; + + bool _isStartNative = false; + + // 新增：SPS/PPS状态追踪变量 + bool hasSps = false; + bool hasPps = false; + + // 新增：SPS/PPS缓存 + List? spsCache; + List? ppsCache; + + void _setupFrameRefresh() { + // 设置帧刷新定时器，16ms对应约60fps + _frameRefreshTimer = + Timer.periodic(const Duration(milliseconds: 16), (timer) { + if (_isFrameAvailable) { + _isFrameAvailable = false; + _renderedFrameCount++; + + // 每秒统计一次帧率 + int now = DateTime.now().millisecondsSinceEpoch; + if (now - _lastRenderedFrameTime > 1000) { + print('[Flutter] 每秒渲染帧数: $_renderedFrameCount'); + _renderedFrameCount = 0; + _lastRenderedFrameTime = now; + } + + // 请求Flutter重建widget + WidgetsBinding.instance.scheduleFrame(); + } + }); + } + + void onFrameAvailable() { + _isFrameAvailable = true; + } + // 初始化视频解码器 Future _initVideoDecoder() async { try { // 创建解码器配置 final config = VideoDecoderConfig( - width: 864, + width: 1280, // 实际视频宽度 - height: 480, - frameRate: 25, - // 明确设置帧率 - // 增大缓冲区大小 - codecType: CodecType.h264, - // 编解码类型 - isDebug: true, + height: 720, + codecType: 'h264', ); - // 初始化解码器并获取textureId final textureId = await VideoDecodePlugin.initDecoder(config); - if (textureId != null) { state.textureId.value = textureId; AppLog.log('视频解码器初始化成功：textureId=$textureId'); - - // 设置帧回调 - VideoDecodePlugin.setFrameCallback(_onFrameAvailable); - - // 设置状态回调 - VideoDecodePlugin.setStateCallbackForTexture( - textureId, _onDecoderStateChanged); - - // 启动FPS监测 - startFpsMonitoring(); + // 启动帧处理定时器 } else { AppLog.log('视频解码器初始化失败'); } + _startFrameProcessTimer(); } catch (e) { AppLog.log('初始化视频解码器错误: $e'); // 如果初始化失败，延迟后重试 @@ -108,66 +144,6 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { } } - // 添加帧可用回调 - void _onFrameAvailable(int textureId) {} - - // 解码器状态变化回调 - void _onDecoderStateChanged( - int textureId, DecoderState decoderState, Map stats) { - String stateText; - switch (decoderState) { - case DecoderState.initializing: - state.isLoading.value = true; - stateText = "初始化中"; - break; - case DecoderState.ready: - stateText = "准备就绪"; - break; - case DecoderState.rendering: - stateText = "渲染中"; - state.isLoading.value = false; - break; - case DecoderState.error: - stateText = "出错"; - // 获取错误信息 - final errorMessage = stats['errorMessage'] as String?; - if (errorMessage != null) { - AppLog.log("解码器错误: $errorMessage"); - } - break; - case DecoderState.released: - stateText = "已释放"; - break; - default: - stateText = "未知状态"; - } - - // 更新统计信息 - if (stats.isNotEmpty) { - // 获取丢包率统计信息 - final PacketLossInfo packetLossInfo = - PacketLossStatistics().getStatistics(); - - // 更新FPS - // state.decoderFps.value = (stats['fps'] as num?)?.toDouble() ?? 0.0; - // 更新解码器详细信息 - state.renderedFrameCount.value = (stats['renderedFrames'] as int?) ?? 0; - state.totalFrames.value = (stats['totalFrames'] as int?) ?? 0; - state.droppedFrames.value = (stats['droppedFrames'] as int?) ?? 0; - state.hasSentIDR.value = (stats['hasSentIDR'] as bool?) ?? false; - state.hasSentSPS.value = (stats['hasSentSPS'] as bool?) ?? false; - state.hasSentPPS.value = (stats['hasSentPPS'] as bool?) ?? false; - state.keyFrameInterval.value = (stats['keyFrameInterval'] as int?) ?? 0; - state.decodingJitterMs.value = (stats['decodingJitterMs'] as int?) ?? 0; - - // 更新状态数据 - state.messageLossRate.value = packetLossInfo.messageLossRate; - state.packetLossRate.value = packetLossInfo.packetLossRate; - state.lastPacketStatsUpdateTime.value = - DateTime.now().millisecondsSinceEpoch; - } - } - /// 初始化音频播放器 void _initFlutterPcmSound() { const int sampleRate = 8000; @@ -190,13 +166,149 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { // 拒绝 StartChartManage().startTalkRejectMessageTimer(); } - if (state.textureId.value != null) { - VideoDecodePlugin.releaseDecoderForTexture(state.textureId.value!); - } - VideoDecodePlugin.releaseAllDecoders(); + VideoDecodePlugin.releaseDecoder(); Get.back(); } + /// 添加H264帧到缓冲区 + void _addFrameToBuffer( + List frameData, + TalkDataH264Frame_FrameTypeE frameType, + int pts, + int frameSeq, + int frameSeqI, + ) { + _networkFrameCount++; + int now = DateTime.now().millisecondsSinceEpoch; + if (now - _lastNetworkPrintTime > 1000) { + AppLog.log('[Flutter] 每秒收到网络H264帧数: ' + _networkFrameCount.toString()); + state.networkH264Fps.value = _networkFrameCount; + _networkFrameCount = 0; + _lastNetworkPrintTime = now; + } + + // 创建包含帧数据和类型的Map + final Map frameMap = { + 'frameData': frameData, + 'frameType': frameType, + 'frameSeq': frameSeq, + 'frameSeqI': frameSeqI, + 'pts': pts, + }; + + // 将帧添加到缓冲区 + state.h264FrameBuffer.add(frameMap); + + // 如果缓冲区超出最大大小，移除最早的帧 + while (state.h264FrameBuffer.length > state.maxFrameBufferSize) { + state.h264FrameBuffer.removeAt(0); + } + + _flutterToNativeFrameCount++; + if (now - _lastFlutterToNativePrintTime > 1000) { + AppLog.log( + '[Flutter] 每秒送入Native帧数: ' + _flutterToNativeFrameCount.toString()); + state.nativeSendFps.value = _flutterToNativeFrameCount; + _flutterToNativeFrameCount = 0; + _lastFlutterToNativePrintTime = now; + } + } + + /// 启动帧处理定时器 + void _startFrameProcessTimer() { + // 取消已有定时器 + state.frameProcessTimer?.cancel(); + + // 计算定时器间隔，确保以目标帧率处理帧 + final int intervalMs = (1000 / state.targetFps).round(); + + // 创建新定时器 + state.frameProcessTimer = + Timer.periodic(Duration(milliseconds: intervalMs), (timer) { + if (state.isLoading.isTrue) { + state.isLoading.value = false; + } + _processNextFrameFromBuffer(); + }); + + AppLog.log('启动帧处理定时器，目标帧率: ${state.targetFps}fps，间隔: ${intervalMs}ms'); + } + + /// 从缓冲区处理下一帧 + void _processNextFrameFromBuffer() async { + // 避免重复处理 + if (state.isProcessingFrame) { + return; + } + + // 如果缓冲区为空，跳过 + if (state.h264FrameBuffer.isEmpty) { + return; + } + + // 设置正在处理标志 + state.isProcessingFrame = true; + + try { + // 取出最早的帧 + final Map frameMap = state.h264FrameBuffer.removeAt(0); + final List frameData = frameMap['frameData']; + final TalkDataH264Frame_FrameTypeE frameType = frameMap['frameType']; + final int frameSeq = frameMap['frameSeq']; + final int frameSeqI = frameMap['frameSeqI']; + int pts = DateTime.now().millisecondsSinceEpoch; + + if (frameType == TalkDataH264Frame_FrameTypeE.P) { + // 以frameSeqI为I帧序号标识 + if (!(_decodedIFrames.contains(frameSeqI))) { + AppLog.log('丢弃P帧：未收到对应I帧，frameSeqI=${frameSeqI}'); + return; + } + } else if (frameType == TalkDataH264Frame_FrameTypeE.I) { + // 记录已解码I帧序号 + _decodedIFrames.add(frameSeq); + } + // 实时写入h264文件 + _appendH264FrameToFile(frameData, frameType); + + final timestamp = DateTime.now().microsecondsSinceEpoch; + VideoDecodePlugin.decodeFrame( + frameData: Uint8List.fromList(frameData), + frameType: frameType == TalkDataH264Frame_FrameTypeE.I ? 1 : 0, + frameSeq: frameSeq, + timestamp: timestamp, + refIFrameSeq: frameSeqI, + ); + + // 判断P帧对应I帧是否已解码，未解码则丢弃P帧 + if (frameType == TalkDataH264Frame_FrameTypeE.P) { + // 以frameSeqI为I帧序号标识 + if (!(_decodedIFrames.contains(frameSeqI))) { + AppLog.log('丢弃P帧：未收到对应I帧，frameSeqI=${frameSeqI}'); + } + } else if (frameType == TalkDataH264Frame_FrameTypeE.I) { + // 记录已解码I帧序号 + _decodedIFrames.add(frameSeq); + } + // 实时写入h264文件 + _appendH264FrameToFile(frameData, frameType); + } catch (e) { + AppLog.log('处理缓冲帧失败: $e'); + } finally { + // 重置处理标志 + state.isProcessingFrame = false; + } + } + + /// 停止帧处理定时器 + void _stopFrameProcessTimer() { + state.frameProcessTimer?.cancel(); + state.frameProcessTimer = null; + state.h264FrameBuffer.clear(); + state.isProcessingFrame = false; + // AppLog.log('停止帧处理定时器'); + } + // 发起接听命令 void initiateAnswerCommand() { StartChartManage().startTalkAcceptTimer(); @@ -230,82 +342,63 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { _playAudioFrames(); break; case TalkData_ContentTypeE.H264: - // 获取当前时间戳 - final currentTimestamp = DateTime.now().millisecondsSinceEpoch; - - // 如果存在上一帧时间戳，则计算时间间隔 - if (_previousFrameTimestamp != null) { - final timeDifference = currentTimestamp - _previousFrameTimestamp!; - AppLog.log('当前帧与上一帧的时间间隔: $timeDifference 毫秒'); - } - // 更新上一帧时间戳为当前帧的时间戳 - _previousFrameTimestamp = currentTimestamp; - - // 添加到视频帧缓冲区，而不是直接处理 - // _processH264Frame(talkData, talkDataH264Frame!); - // 直接处理H264视频帧 - // _processH264Frame(talkData, talkDataH264Frame!); - // - // // 记录关键调试信息 - // if (talkDataH264Frame!.frameType == TalkDataH264Frame_FrameTypeE.I) { - // AppLog.log( - // '帧序号${talkDataH264Frame.frameSeq};帧类型:${talkDataH264Frame.frameType.toString()};时间戳:${DateTime.now().millisecondsSinceEpoch}'); + // if (_isStartNative) { + // if (talkDataH264Frame != null) { + // // 如果是I帧，先分割NALU，找到SPS/PPS并优先放入缓冲区 + // if (talkDataH264Frame.frameType == + // TalkDataH264Frame_FrameTypeE.I) { + // // 清空缓冲区，丢弃I帧前所有未处理帧（只保留SPS/PPS/I帧） + // state.h264FrameBuffer.clear(); + // _extractAndBufferSpsPpsForBuffer( + // talkData.content, + // talkData.durationMs, + // talkDataH264Frame.frameSeq, + // talkDataH264Frame.frameSeqI); + // } + // _addFrameToBuffer( + // talkData.content, + // talkDataH264Frame.frameType, + // talkData.durationMs, + // talkDataH264Frame.frameSeq, + // talkDataH264Frame.frameSeqI); + // } + // } else { + // await VideoDecodePlugin.startNativePlayer( + // VideoDecoderConfig(width: 1280, height: 720, codecType: 'h264'), + // ); + // _isStartNative = true; // } + // 处理H264帧 + if (state.textureId.value != null) { + if (talkDataH264Frame != null) { + if (talkDataH264Frame.frameType == + TalkDataH264Frame_FrameTypeE.I) { + _handleIFrameWithSpsPpsAndIdr( + talkData.content, + talkData.durationMs, + talkDataH264Frame.frameSeq, + talkDataH264Frame.frameSeqI, + ); + return; + } else if (talkDataH264Frame.frameType == + TalkDataH264Frame_FrameTypeE.P) { + _handlePFrame( + talkData.content, + talkData.durationMs, + talkDataH264Frame.frameSeq, + talkDataH264Frame.frameSeqI, + ); + return; + } + } + } else { + AppLog.log('无法处理H264帧：textureId为空'); + } break; } }); } - // 处理H264视频帧 - Future _processH264Frame( - TalkData talkData, TalkDataH264Frame frameInfo) async { - // 检查解码器是否已初始化 - if (state.textureId.value == null) { - // 可以记录日志或尝试初始化解码器 - AppLog.log('解码器尚未初始化，尝试重新初始化...'); - await _initVideoDecoder(); - - // 如果仍未初始化成功，则丢弃此帧 - if (state.textureId.value == null) { - return; - } - } - // 获取P帧对应的I帧序号 - final frameSeqI = frameInfo.frameSeqI; - - // P帧检查：如果依赖的I帧未解码成功，直接丢弃 - if (frameInfo.frameType == TalkDataH264Frame_FrameTypeE.P && - !_decodedIFrames.contains(frameSeqI)) { - AppLog.log('丢弃P帧: 依赖的I帧(${frameSeqI})尚未解码, P帧序号: ${frameInfo.frameSeq}'); - return; - } - - // 从talkData中提取H264帧数据 - final Uint8List frameData = Uint8List.fromList(talkData.content); - - // 确定帧类型 - final FrameType frameType = - frameInfo.frameType == TalkDataH264Frame_FrameTypeE.I - ? FrameType.iFrame - : FrameType.pFrame; - - // 将帧数据交给解码器处理 - try { - final bool result = - await VideoDecodePlugin.decodeFrame(frameData, frameType); - // 如果是I帧且成功解码，将其序号加入已解码I帧集合 - if (frameInfo.frameType == TalkDataH264Frame_FrameTypeE.I && result) { - _decodedIFrames.add(frameInfo.frameSeq); - // 限制集合大小，避免内存泄漏 - if (_decodedIFrames.length > 30) { - _decodedIFrames.remove(_decodedIFrames.first); - } - } - } catch (e) { - AppLog.log('解码帧错误: $e, 帧序号: ${frameInfo.frameSeq}'); - } - } - // 新增：音频帧播放逻辑 void _playAudioFrames() { // 如果缓冲区为空或未达到目标大小，不进行播放 @@ -462,10 +555,20 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { // 初始化视频解码器 _initVideoDecoder(); + + // 初始化H264帧缓冲区 + state.h264FrameBuffer.clear(); + state.isProcessingFrame = false; + + _setupFrameRefresh(); } @override void onClose() { + _closeH264File(); + // 停止帧处理定时器 + _stopFrameProcessTimer(); + _stopPlayG711Data(); // 停止播放音频 state.audioBuffer.clear(); // 清空音频缓冲区 @@ -490,11 +593,9 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { _streamSubscription?.cancel(); _isListening = false; - // 停止FPS监测 - stopFpsMonitoring(); // 重置期望数据 StartChartManage().reSetDefaultTalkExpect(); - VideoDecodePlugin.releaseAllDecoders(); + VideoDecodePlugin.releaseDecoder(); // 取消批处理定时器 _batchProcessTimer?.cancel(); @@ -502,7 +603,8 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { // 清空已解码I帧集合 _decodedIFrames.clear(); - + _frameRefreshTimer?.cancel(); + _frameRefreshTimer = null; super.onClose(); } @@ -722,91 +824,430 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { return result; } - // 启动网络状态监测 - void startFpsMonitoring() { - // 确保只有一个计时器在运行 - stopFpsMonitoring(); - - // 初始化时间记录 - state.lastFpsUpdateTime.value = DateTime.now().millisecondsSinceEpoch; - - // 创建一个计时器，每秒更新一次丢包率和性能数据 - state.fpsTimer = Timer.periodic(const Duration(seconds: 1), (timer) { - // 更新丢包率数据 - updatePacketLossStats(); - - // 分析性能数据 - _analyzePerformance(); - }); - } - - // 停止网络状态监测 - void stopFpsMonitoring() { - state.fpsTimer?.cancel(); - state.fpsTimer = null; - } - - // 日志记录方法 - void logMessage(String message) { - AppLog.log(message); - } - - // 更新丢包率统计数据 - void updatePacketLossStats() async { - try {} catch (e) { - logMessage('获取丢包率数据失败: $e'); - } - } - - // 添加性能分析方法 - void _analyzePerformance() { - final int now = DateTime.now().millisecondsSinceEpoch; - - // 如果是首次调用，初始化数据 - if (state.lastPerformanceCheck == 0) { - state.lastPerformanceCheck = now; - state.lastFrameCount = state.renderedFrameCount.value; - return; - } - - // 每秒分析一次性能 - if (now - state.lastPerformanceCheck >= 1000) { - // 计算过去一秒的实际帧率 - final int frameRendered = - state.renderedFrameCount.value - state.lastFrameCount; - final double actualFPS = - frameRendered * 1000 / (now - state.lastPerformanceCheck); - - // 计算丢帧率 - final double dropRate = state.droppedFrames.value / - (state.totalFrames.value > 0 ? state.totalFrames.value : 1) * - 100; - - // 计算当前解码器积压帧数 - final int pendingFrames = - state.totalFrames.value - state.renderedFrameCount.value; - - // 计算跟踪Map中的帧数（正在处理中的帧） - final int processingFrames = state.frameTracker.length; - - // 分析渲染瓶颈 - String performanceStatus = "正常"; - if (actualFPS < 15 && dropRate > 10) { - performanceStatus = "严重渲染瓶颈"; - } else if (actualFPS < 20 && dropRate > 5) { - performanceStatus = "轻微渲染瓶颈"; + /// 追加写入一帧到h264文件（需传入帧数据和帧类型frameType） + Future _appendH264FrameToFile( + List frameData, TalkDataH264Frame_FrameTypeE frameType) async { + try { + if (state.h264File == null) { + await _initH264File(); + } + // NALU分割函数，返回每个NALU的完整字节数组 + List> splitNalus(List data) { + List> nalus = []; + int i = 0; + while (i < data.length - 3) { + int start = -1; + int next = -1; + if (data[i] == 0x00 && data[i + 1] == 0x00) { + if (data[i + 2] == 0x01) { + start = i; + i += 3; + } else if (i + 3 < data.length && + data[i + 2] == 0x00 && + data[i + 3] == 0x01) { + start = i; + i += 4; + } else { + i++; + continue; + } + next = i; + while (next < data.length - 3) { + if (data[next] == 0x00 && + data[next + 1] == 0x00 && + ((data[next + 2] == 0x01) || + (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { + break; + } + next++; + } + nalus.add(data.sublist(start, next)); + i = next; + } else { + i++; + } + } + int nalusTotalLen = + nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; + if (nalus.isEmpty && data.isNotEmpty) { + nalus.add(data); + } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { + nalus.add(data.sublist(nalusTotalLen)); + } + return nalus; } - // 输出综合性能分析 - AppLog.log("性能分析: 实际帧率=${actualFPS.toStringAsFixed(1)}fps, " + - "丢帧率=${dropRate.toStringAsFixed(1)}%, " + - "待处理帧数=$pendingFrames, " + - "处理中帧数=$processingFrames, " + - "状态=$performanceStatus"); + // 优化：I帧前只缓存SPS/PPS/IDR，首次写入严格顺序 + if (!_hasWrittenFirstIFrame) { + final nalus = splitNalus(frameData); + List> spsList = []; + List> ppsList = []; + List> idrList = []; + for (final nalu in nalus) { + int offset = 0; + if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { + if (nalu[2] == 0x01) + offset = 3; + else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; + } + if (nalu.length > offset) { + int naluType = nalu[offset] & 0x1F; + if (naluType == 7) { + spsList.add(nalu); + AppLog.log('SPS内容: ' + + nalu + .map((b) => b.toRadixString(16).padLeft(2, '0')) + .join(' ')); + } else if (naluType == 8) { + ppsList.add(nalu); + AppLog.log('PPS内容: ' + + nalu + .map((b) => b.toRadixString(16).padLeft(2, '0')) + .join(' ')); + } else if (naluType == 5) { + idrList.add(nalu); + } + // 其他类型不缓存也不写入头部 + } + } + // 只在首次I帧写入前缓存，严格顺序写入 + if (spsList.isNotEmpty && ppsList.isNotEmpty && idrList.isNotEmpty) { + for (final sps in spsList) { + await _writeSingleFrameToFile(_ensureStartCode(sps)); + AppLog.log('写入顺序: SPS'); + } + for (final pps in ppsList) { + await _writeSingleFrameToFile(_ensureStartCode(pps)); + AppLog.log('写入顺序: PPS'); + } + for (final idr in idrList) { + await _writeSingleFrameToFile(_ensureStartCode(idr)); + AppLog.log('写入顺序: IDR'); + } + _hasWrittenFirstIFrame = true; + } else { + // 未收齐SPS/PPS/IDR则继续缓存，等待下次I帧 + if (spsList.isNotEmpty) _preIFrameCache.addAll(spsList); + if (ppsList.isNotEmpty) _preIFrameCache.addAll(ppsList); + if (idrList.isNotEmpty) _preIFrameCache.addAll(idrList); + } + } else { + // 首帧后只写入IDR和P帧 + final nalus = splitNalus(frameData); + for (final nalu in nalus) { + int offset = 0; + if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { + if (nalu[2] == 0x01) + offset = 3; + else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; + } + if (nalu.length > offset) { + int naluType = nalu[offset] & 0x1F; + if (naluType == 5) { + await _writeSingleFrameToFile(_ensureStartCode(nalu)); + // AppLog.log('写入顺序: IDR'); + } else if (naluType == 1) { + await _writeSingleFrameToFile(_ensureStartCode(nalu)); + // AppLog.log('写入顺序: P帧'); + } else if (naluType == 7) { + // AppLog.log('遇到新SPS，已忽略'); + } else if (naluType == 8) { + // AppLog.log('遇到新PPS，已忽略'); + } + // 其他类型不写入 + } + } + } + } catch (e) { + AppLog.log('写入H264帧到文件失败: $e'); + } + } - // 重置统计数据 - state.lastPerformanceCheck = now; - state.lastFrameCount = state.renderedFrameCount.value; + // 统一NALU起始码为0x00000001 + List _ensureStartCode(List nalu) { + if (nalu.length >= 4 && + nalu[0] == 0x00 && + nalu[1] == 0x00 && + nalu[2] == 0x00 && + nalu[3] == 0x01) { + return nalu; + } else if (nalu.length >= 3 && + nalu[0] == 0x00 && + nalu[1] == 0x00 && + nalu[2] == 0x01) { + return [0x00, 0x00, 0x00, 0x01] + nalu.sublist(3); + } else { + return [0x00, 0x00, 0x00, 0x01] + nalu; + } + } + + /// 实际写入单帧到文件（带NALU头判断） + Future _writeSingleFrameToFile(List frameData) async { + bool hasNaluHeader = false; + if (frameData.length >= 4 && + frameData[0] == 0x00 && + frameData[1] == 0x00 && + ((frameData[2] == 0x01) || + (frameData[2] == 0x00 && frameData[3] == 0x01))) { + hasNaluHeader = true; + } + if (hasNaluHeader) { + await state.h264File!.writeAsBytes(frameData, mode: FileMode.append); + } else { + final List naluHeader = [0x00, 0x00, 0x01]; + await state.h264File! + .writeAsBytes(naluHeader + frameData, mode: FileMode.append); + } + } + + /// 初始化h264文件 + Future _initH264File() async { + try { + if (state.h264File != null) return; + // 获取Download目录 + Directory? downloadsDir; + if (Platform.isAndroid) { + // Android 10+ 推荐用getExternalStorageDirectory() + downloadsDir = await getExternalStorageDirectory(); + // 兼容部分ROM，优先用Download + final downloadPath = '/storage/emulated/0/Download'; + if (Directory(downloadPath).existsSync()) { + downloadsDir = Directory(downloadPath); + } + } else { + downloadsDir = await getApplicationDocumentsDirectory(); + } + final filePath = + '${downloadsDir!.path}/video_${DateTime.now().millisecondsSinceEpoch}.h264'; + state.h264FilePath = filePath; + state.h264File = File(filePath); + if (!await state.h264File!.exists()) { + await state.h264File!.create(recursive: true); + } + AppLog.log('H264文件初始化: $filePath'); + } catch (e) { + AppLog.log('H264文件初始化失败: $e'); + } + } + + /// 关闭h264文件 + Future _closeH264File() async { + try { + if (state.h264File != null) { + AppLog.log('H264文件已关闭: ${state.h264FilePath ?? ''}'); + } + state.h264File = null; + state.h264FilePath = null; + _preIFrameCache.clear(); + _hasWrittenFirstIFrame = false; + } catch (e) { + AppLog.log('关闭H264文件时出错: $e'); + } + } + + /// 从I帧数据中分割NALU并将SPS/PPS优先放入缓冲区（用于缓冲区发送） + void _extractAndBufferSpsPpsForBuffer( + List frameData, int durationMs, int frameSeq, int frameSeqI) { + List> splitNalus(List data) { + List> nalus = []; + int i = 0; + while (i < data.length - 3) { + int start = -1; + int next = -1; + if (data[i] == 0x00 && data[i + 1] == 0x00) { + if (data[i + 2] == 0x01) { + start = i; + i += 3; + } else if (i + 3 < data.length && + data[i + 2] == 0x00 && + data[i + 3] == 0x01) { + start = i; + i += 4; + } else { + i++; + continue; + } + next = i; + while (next < data.length - 3) { + if (data[next] == 0x00 && + data[next + 1] == 0x00 && + ((data[next + 2] == 0x01) || + (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { + break; + } + next++; + } + nalus.add(data.sublist(start, next)); + i = next; + } else { + i++; + } + } + int nalusTotalLen = + nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; + if (nalus.isEmpty && data.isNotEmpty) { + nalus.add(data); + } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { + nalus.add(data.sublist(nalusTotalLen)); + } + return nalus; + } + + final nalus = splitNalus(frameData); + for (final nalu in nalus) { + int offset = 0; + if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { + if (nalu[2] == 0x01) + offset = 3; + else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; + } + if (nalu.length > offset) { + int naluType = nalu[offset] & 0x1F; + if (naluType == 7) { + // SPS + hasSps = true; + // 只在首次或内容变化时更新缓存 + if (spsCache == null || !_listEquals(spsCache!, nalu)) { + spsCache = List.from(nalu); + } + } else if (naluType == 8) { + // PPS + hasPps = true; + if (ppsCache == null || !_listEquals(ppsCache!, nalu)) { + ppsCache = List.from(nalu); + } + } + } + } + } + + // 新增：List比较工具 + bool _listEquals(List a, List b) { + if (a.length != b.length) return false; + for (int i = 0; i < a.length; i++) { + if (a[i] != b[i]) return false; + } + return true; + } + + // 新增：I帧处理方法 + void _handleIFrameWithSpsPpsAndIdr(List frameData, int durationMs, int frameSeq, int frameSeqI) { + // 清空缓冲区，丢弃I帧前所有未处理帧（只保留SPS/PPS/I帧） + state.h264FrameBuffer.clear(); + _extractAndBufferSpsPpsForBuffer(frameData, durationMs, frameSeq, frameSeqI); + // 只要缓存有SPS/PPS就先写入，再写I帧本体（只写IDR） + if (spsCache == null || ppsCache == null) { + // 没有SPS/PPS缓存，丢弃本次I帧 + return; + } + // 先写入SPS/PPS + _addFrameToBuffer(spsCache!, TalkDataH264Frame_FrameTypeE.I, durationMs, frameSeq, frameSeqI); + _addFrameToBuffer(ppsCache!, TalkDataH264Frame_FrameTypeE.I, durationMs, frameSeq, frameSeqI); + // 分割I帧包，只写入IDR（type 5） + List> nalus = []; + int i = 0; + List data = frameData; + while (i < data.length - 3) { + int start = -1; + int next = -1; + if (data[i] == 0x00 && data[i + 1] == 0x00) { + if (data[i + 2] == 0x01) { + start = i; + i += 3; + } else if (i + 3 < data.length && data[i + 2] == 0x00 && data[i + 3] == 0x01) { + start = i; + i += 4; + } else { + i++; + continue; + } + next = i; + while (next < data.length - 3) { + if (data[next] == 0x00 && data[next + 1] == 0x00 && ((data[next + 2] == 0x01) || (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { + break; + } + next++; + } + nalus.add(data.sublist(start, next)); + i = next; + } else { + i++; + } + } + int nalusTotalLen = nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; + if (nalus.isEmpty && data.isNotEmpty) { + nalus.add(data); + } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { + nalus.add(data.sublist(nalusTotalLen)); + } + for (final nalu in nalus) { + int offset = 0; + if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { + if (nalu[2] == 0x01) + offset = 3; + else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; + } + if (nalu.length > offset) { + int naluType = nalu[offset] & 0x1F; + if (naluType == 5) { + _addFrameToBuffer(nalu, TalkDataH264Frame_FrameTypeE.I, durationMs, frameSeq, frameSeqI); + } + } + } + } + + // 新增：P帧处理方法 + void _handlePFrame(List frameData, int durationMs, int frameSeq, int frameSeqI) { + // 只写入P帧（type 1） + List> nalus = []; + int i = 0; + List data = frameData; + while (i < data.length - 3) { + int start = -1; + int next = -1; + if (data[i] == 0x00 && data[i + 1] == 0x00) { + if (data[i + 2] == 0x01) { + start = i; + i += 3; + } else if (i + 3 < data.length && data[i + 2] == 0x00 && data[i + 3] == 0x01) { + start = i; + i += 4; + } else { + i++; + continue; + } + next = i; + while (next < data.length - 3) { + if (data[next] == 0x00 && data[next + 1] == 0x00 && ((data[next + 2] == 0x01) || (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { + break; + } + next++; + } + nalus.add(data.sublist(start, next)); + i = next; + } else { + i++; + } + } + int nalusTotalLen = nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; + if (nalus.isEmpty && data.isNotEmpty) { + nalus.add(data); + } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { + nalus.add(data.sublist(nalusTotalLen)); + } + for (final nalu in nalus) { + int offset = 0; + if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { + if (nalu[2] == 0x01) + offset = 3; + else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; + } + if (nalu.length > offset) { + int naluType = nalu[offset] & 0x1F; + if (naluType == 1) { + _addFrameToBuffer(nalu, TalkDataH264Frame_FrameTypeE.P, durationMs, frameSeq, frameSeqI); + } + } } } } diff --git a/lib/talk/starChart/views/native/talk_view_native_decode_page.dart b/lib/talk/starChart/views/native/talk_view_native_decode_page.dart index 3172a6a6..a5c69b15 100644 --- a/lib/talk/starChart/views/native/talk_view_native_decode_page.dart +++ b/lib/talk/starChart/views/native/talk_view_native_decode_page.dart @@ -56,6 +56,7 @@ class _TalkViewNativeDecodePageState extends State state.animationController.forward(); } }); + } @override @@ -71,6 +72,7 @@ class _TalkViewNativeDecodePageState extends State child: Stack( alignment: Alignment.center, children: [ + // 悬浮帧率统计信息条 Obx( () { final double screenWidth = MediaQuery.of(context).size.width; @@ -93,32 +95,84 @@ class _TalkViewNativeDecodePageState extends State final double scaleWidth = physicalWidth / rotatedImageWidth; final double scaleHeight = physicalHeight / rotatedImageHeight; max(scaleWidth, scaleHeight); // 选择较大的缩放比例 - - return state.isLoading.isTrue - ? Image.asset( - 'images/main/monitorBg.png', - width: screenWidth, - height: screenHeight, - fit: BoxFit.cover, - ) - : PopScope( - canPop: false, - child: RepaintBoundary( - key: state.globalKey, - child: SizedBox.expand( - child: RotatedBox( - // 解码器不支持硬件旋转，使用RotatedBox - quarterTurns: -1, - child: Texture( - textureId: state.textureId.value!, - filterQuality: FilterQuality.medium, + return Column( + children: [ + Expanded( + child: state.isLoading.isTrue + ? Image.asset( + 'images/main/monitorBg.png', + width: screenWidth, + height: screenHeight, + fit: BoxFit.cover, + ) + : PopScope( + canPop: false, + child: RepaintBoundary( + key: state.globalKey, + child: SizedBox.expand( + child: RotatedBox( + // 解码器不支持硬件旋转，使用RotatedBox + quarterTurns: -1, + child: Texture( + textureId: state.textureId.value!, + filterQuality: FilterQuality.medium, + ), + ), + ), ), ), - ), - ), - ); + ), + ], + ); }, ), + Positioned( + top: 300.h, + right: 20.w, + child: Obx(() => Container( + padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 10), + decoration: BoxDecoration( + color: Colors.black.withOpacity(0.5), + borderRadius: BorderRadius.circular(12), + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + children: [ + Icon(Icons.network_check, color: Colors.redAccent, size: 18), + SizedBox(width: 6), + Text( + '接受服务端H264帧率/秒: ', + style: TextStyle(color: Colors.white, fontSize: 15), + ), + Text( + '${state.networkH264Fps.value}', + style: TextStyle(color: Colors.redAccent, fontSize: 16, fontWeight: FontWeight.bold), + ), + Text(' fps', style: TextStyle(color: Colors.white, fontSize: 13)), + ], + ), + SizedBox(height: 4), + Row( + children: [ + Icon(Icons.send, color: Colors.blueAccent, size: 18), + SizedBox(width: 6), + Text( + '送入Native帧率/秒: ', + style: TextStyle(color: Colors.white, fontSize: 15), + ), + Text( + '${state.nativeSendFps.value}', + style: TextStyle(color: Colors.blueAccent, fontSize: 16, fontWeight: FontWeight.bold), + ), + Text(' fps', style: TextStyle(color: Colors.white, fontSize: 13)), + ], + ), + ], + ), + )), + ), Obx(() => state.isLoading.isTrue ? Positioned( bottom: 310.h, @@ -127,102 +181,6 @@ class _TalkViewNativeDecodePageState extends State style: TextStyle(color: Colors.black, fontSize: 26.sp), )) : Container()), - Obx(() => state.textureId.value != null && state.showFps.value - ? Positioned( - top: ScreenUtil().statusBarHeight + 10.h, - right: 20.w, - child: Container( - padding: - EdgeInsets.symmetric(horizontal: 10.w, vertical: 5.h), - decoration: BoxDecoration( - color: Colors.black.withOpacity(0.5), - borderRadius: BorderRadius.circular(5.h), - ), - child: Column( - crossAxisAlignment: CrossAxisAlignment.end, - children: [ - // Text( - // 'FPS: ${state.decoderFps.value.toStringAsFixed(1)}', - // style: TextStyle( - // color: _getPacketLossColor( - // state.packetLossRate.value), - // fontSize: 20.sp, - // ), - // ), - Text( - '丢包率: ${state.packetLossRate.value.toStringAsFixed(1)}%', - style: TextStyle( - color: _getPacketLossColor( - state.packetLossRate.value), - fontSize: 20.sp, - ), - ), - Text( - '消息丢失: ${state.messageLossRate.value.toStringAsFixed(1)}%', - style: TextStyle( - color: _getPacketLossColor( - state.messageLossRate.value), - fontSize: 20.sp, - ), - ), - Divider( - color: Colors.white30, - height: 10.h, - thickness: 1), - Text( - '已渲染帧: ${state.renderedFrameCount.value}', - style: - TextStyle(color: Colors.white, fontSize: 18.sp), - ), - Text( - '总帧数: ${state.totalFrames.value}', - style: - TextStyle(color: Colors.white, fontSize: 18.sp), - ), - Text( - '丢弃帧: ${state.droppedFrames.value}', - style: - TextStyle(color: Colors.white, fontSize: 18.sp), - ), - Text( - 'IDR帧: ${state.hasSentIDR.value ? "已发送" : "未发送"}', - style: TextStyle( - color: state.hasSentIDR.value - ? Colors.green - : Colors.red, - fontSize: 18.sp), - ), - Text( - 'SPS: ${state.hasSentSPS.value ? "已发送" : "未发送"}', - style: TextStyle( - color: state.hasSentSPS.value - ? Colors.green - : Colors.red, - fontSize: 18.sp), - ), - Text( - 'PPS: ${state.hasSentPPS.value ? "已发送" : "未发送"}', - style: TextStyle( - color: state.hasSentPPS.value - ? Colors.green - : Colors.red, - fontSize: 18.sp), - ), - Text( - 'keyFrameInterval: ${state.keyFrameInterval.value}', - style: - TextStyle(color: Colors.green, fontSize: 18.sp), - ), - Text( - 'decodingJitterMs: ${state.decodingJitterMs.value}', - style: - TextStyle(color: Colors.green, fontSize: 18.sp), - ), - ], - ), - ), - ) - : Container()), Obx(() => state.isLoading.isFalse && state.oneMinuteTime.value > 0 ? Positioned( top: ScreenUtil().statusBarHeight + 75.h, diff --git a/lib/talk/starChart/views/native/talk_view_native_decode_state.dart b/lib/talk/starChart/views/native/talk_view_native_decode_state.dart index d98d5d34..6eef401e 100644 --- a/lib/talk/starChart/views/native/talk_view_native_decode_state.dart +++ b/lib/talk/starChart/views/native/talk_view_native_decode_state.dart @@ -106,4 +106,19 @@ class TalkViewNativeDecodeState { // 帧跟踪Map，记录每个提交的帧，key为textureId_frameSeq Map> frameTracker = {}; + + // H264帧缓冲区相关 + final List> h264FrameBuffer = >[]; // H264帧缓冲区，存储帧数据和类型 + final int maxFrameBufferSize = 25; // 最大缓冲区大小 + final int targetFps = 120; // 目标解码帧率 + Timer? frameProcessTimer; // 帧处理定时器 + bool isProcessingFrame = false; // 是否正在处理帧 + int lastProcessedTimestamp = 0; // 上次处理帧的时间戳 + // H264文件保存相关 + String? h264FilePath; + File? h264File; + + // 新增：用于页面显示的帧率统计 + RxInt networkH264Fps = 0.obs; + RxInt nativeSendFps = 0.obs; }