diff --git a/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart b/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart index 028ffb73..702072b4 100644 --- a/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart +++ b/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart @@ -16,6 +16,11 @@ import 'package:star_lock/talk/starChart/proto/talk_data_h264_frame.pb.dart'; // implements ScpMessageHandler { class UdpTalkDataHandler extends ScpMessageBaseHandle implements ScpMessageHandler { + // 单例实现 + static final UdpTalkDataHandler instance = UdpTalkDataHandler(); + + UdpTalkDataHandler(); // 保持默认构造函数 + @override void handleReq(ScpMessage scpMessage) {} @@ -32,8 +37,11 @@ class UdpTalkDataHandler extends ScpMessageBaseHandle if (scpMessage.Payload != null) { final TalkData talkData = scpMessage.Payload; - // 处理音视频数据 - _handleTalkData(talkData: talkData); + + _handleTalkData( + talkData: talkData, + scpMessage: scpMessage, + ); } } @@ -93,12 +101,15 @@ class UdpTalkDataHandler extends ScpMessageBaseHandle return hexList.join(''); } - void _handleTalkData({required TalkData talkData}) { + void _handleTalkData({ + required TalkData talkData, + required ScpMessage scpMessage, + }) { if (talkData == null) return; final contentType = talkData.contentType; switch (contentType) { case TalkData_ContentTypeE.H264: - _handleVideoH264(talkData); + _handleVideoH264(talkData, scpMessage); break; case TalkData_ContentTypeE.Image: _handleVideoImage(talkData); @@ -113,10 +124,12 @@ class UdpTalkDataHandler extends ScpMessageBaseHandle } /// 处理h264协议的数据 - void _handleVideoH264(TalkData talkData) { + void _handleVideoH264(TalkData talkData, ScpMessage scpMessage) { final TalkDataH264Frame talkDataH264Frame = TalkDataH264Frame(); talkDataH264Frame.mergeFromBuffer(talkData.content); - frameHandler.handleFrame(talkDataH264Frame, talkData); + // AppLog.log('处理H264帧: frameType=${talkDataH264Frame.frameType}, frameSeq=${talkDataH264Frame.frameSeq},MessageId:${scpMessage.MessageId}'); + frameHandler.handleFrame(talkDataH264Frame, talkData, scpMessage); + } /// 处理图片数据 diff --git a/lib/talk/starChart/handle/impl/udp_talk_expect_handler.dart b/lib/talk/starChart/handle/impl/udp_talk_expect_handler.dart index c592b454..ed975f57 100644 --- a/lib/talk/starChart/handle/impl/udp_talk_expect_handler.dart +++ b/lib/talk/starChart/handle/impl/udp_talk_expect_handler.dart @@ -93,20 +93,21 @@ class UdpTalkExpectHandler extends ScpMessageBaseHandle } if (isWifiLockType || (talkExpectResp.rotate == 0 && - talkExpectResp.width == 640 && - talkExpectResp.height == 480)) { + talkExpectResp.width == 640 && + talkExpectResp.height == 480) && + talkStatus.status != TalkStatus.answeredSuccessfully) { Get.toNamed(Routers.imageTransmissionView); - return; } if (startChartManage - .getDefaultTalkExpect() - .videoType - .contains(VideoTypeE.H264)) { + .getDefaultTalkExpect() + .videoType + .contains(VideoTypeE.H264) && + talkStatus.status != TalkStatus.answeredSuccessfully) { Get.toNamed( Routers.h264View, ); - } else { + } else if (talkStatus.status != TalkStatus.answeredSuccessfully) { Get.toNamed( Routers.starChartTalkView, ); diff --git a/lib/talk/starChart/handle/other/h264_frame_handler.dart b/lib/talk/starChart/handle/other/h264_frame_handler.dart index face6b68..018af2a4 100644 --- a/lib/talk/starChart/handle/other/h264_frame_handler.dart +++ b/lib/talk/starChart/handle/other/h264_frame_handler.dart @@ -3,6 +3,7 @@ import 'dart:typed_data'; import 'package:flutter/services.dart'; import 'package:star_lock/app_settings/app_settings.dart'; +import 'package:star_lock/talk/starChart/entity/scp_message.dart'; import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; import '../../proto/talk_data_h264_frame.pb.dart'; @@ -12,8 +13,15 @@ class H264FrameHandler { H264FrameHandler({required this.onCompleteFrame}); - void handleFrame(TalkDataH264Frame frame, TalkData talkData) { - onCompleteFrame( - TalkDataModel(talkData: talkData, talkDataH264Frame: frame)); + void handleFrame( + TalkDataH264Frame frame, TalkData talkData, ScpMessage scpMessage) { + // AppLog.log( + // '送入stream的帧数据: frameSeq=${frame.frameSeq},frameType=${frame + // .frameType},MessageId:${scpMessage.MessageId}'); + onCompleteFrame(TalkDataModel( + talkData: talkData, + talkDataH264Frame: frame, + scpMessage: scpMessage, + )); } } diff --git a/lib/talk/starChart/handle/other/talk_data_model.dart b/lib/talk/starChart/handle/other/talk_data_model.dart index a84f1d3d..f40f44d7 100644 --- a/lib/talk/starChart/handle/other/talk_data_model.dart +++ b/lib/talk/starChart/handle/other/talk_data_model.dart @@ -1,9 +1,12 @@ +import 'package:star_lock/talk/starChart/entity/scp_message.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pbserver.dart'; import 'package:star_lock/talk/starChart/proto/talk_data_h264_frame.pb.dart'; class TalkDataModel { TalkData? talkData; TalkDataH264Frame? talkDataH264Frame; + ScpMessage? scpMessage; - TalkDataModel({required this.talkData, this.talkDataH264Frame}); + TalkDataModel( + {required this.talkData, this.talkDataH264Frame, this.scpMessage}); } diff --git a/lib/talk/starChart/handle/other/talk_data_repository.dart b/lib/talk/starChart/handle/other/talk_data_repository.dart index 587a481f..0d73ed83 100644 --- a/lib/talk/starChart/handle/other/talk_data_repository.dart +++ b/lib/talk/starChart/handle/other/talk_data_repository.dart @@ -11,7 +11,7 @@ class TalkDataRepository { onCancel: () { _isListening = false; }, - sync: false, // 改为同步模式以提高实时性 + sync: true, // 改为同步模式以提高实时性 ); } diff --git a/lib/talk/starChart/handle/scp_message_base_handle.dart b/lib/talk/starChart/handle/scp_message_base_handle.dart index 495db676..d16e5e58 100644 --- a/lib/talk/starChart/handle/scp_message_base_handle.dart +++ b/lib/talk/starChart/handle/scp_message_base_handle.dart @@ -181,40 +181,6 @@ class ScpMessageBaseHandle { } return null; - - // if (!_packetBuffer.containsKey(key)) { - // _packetBuffer[key] = List.filled(spTotal, []); - // _startTimer(key); - // } - // - // // 检查分包索引是否在合法范围内 - // if (spIndex < 1 || spIndex > spTotal) { - // // print( - // // 'Invalid spTotal: $spTotal spIndex: $spIndex for messageId: $messageId'); - // return null; - // } - // - // // 存储当前分包 - // _packetBuffer[key]![spIndex - 1] = byte; - // - // // 检查是否接收到所有分包 - // if (_packetBuffer[key]!.every((packet) => packet.isNotEmpty)) { - // // 重组所有分包 - // Uint8List completePayload = Uint8List.fromList( - // _packetBuffer[key]!.expand((packet) => packet).toList()); - // // 清除已重组和超时的分包数据 - // _clearPacketData(key); - // - // // 使用重组的包构造成TalkData - // if (payloadType == PayloadTypeConstant.talkData) { - // final talkData = TalkData(); - // talkData.mergeFromBuffer(completePayload); - // return talkData; - // } - // } else { - // // 如果分包尚未接收完全，返回 null 或其他指示符 - // return null; - // } } // 启动定时器 diff --git a/lib/talk/starChart/handle/scp_message_handler_factory.dart b/lib/talk/starChart/handle/scp_message_handler_factory.dart index 67c7a963..fabf7429 100644 --- a/lib/talk/starChart/handle/scp_message_handler_factory.dart +++ b/lib/talk/starChart/handle/scp_message_handler_factory.dart @@ -52,7 +52,7 @@ class ScpMessageHandlerFactory { case PayloadTypeConstant.talkExpect: return UdpTalkExpectHandler(); case PayloadTypeConstant.talkData: - return UdpTalkDataHandler(); + return UdpTalkDataHandler.instance; case PayloadTypeConstant.talkHangup: return UdpTalkHangUpHandler(); case PayloadTypeConstant.RbcuInfo: diff --git a/lib/talk/starChart/star_chart_manage.dart b/lib/talk/starChart/star_chart_manage.dart index 7a185c47..81d36ccc 100644 --- a/lib/talk/starChart/star_chart_manage.dart +++ b/lib/talk/starChart/star_chart_manage.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'dart:io'; +import 'dart:isolate'; import 'dart:math'; import 'dart:typed_data'; @@ -50,6 +51,10 @@ import 'package:star_lock/tools/deviceInfo_utils.dart'; import 'package:star_lock/tools/storage.dart'; import 'package:uuid/uuid.dart'; +// Socket选项常量 +const int SO_RCVBUF = 8; // 接收缓冲区 +const int SO_SNDBUF = 7; // 发送缓冲区 + class StartChartManage { // 私有构造函数，防止外部直接new对象 StartChartManage._internal(); @@ -125,6 +130,17 @@ class StartChartManage { // 获取 StartChartTalkStatus 的唯一实例 StartChartTalkStatus talkStatus = StartChartTalkStatus.instance; + // 音视频帧级丢包统计变量 + final Map> _avFrameParts = {}; + int _avFrameTotal = 0; + int _avFrameLost = 0; + + // 查询音视频帧丢包率 + double getAvFrameLossRate() { + if (_avFrameTotal == 0) return 0.0; + return _avFrameLost / _avFrameTotal; + } + // 星图服务初始化 Future init() async { if (F.isXHJ) { @@ -225,6 +241,24 @@ class StartChartManage { var addressIListenFrom = InternetAddress.anyIPv4; RawDatagramSocket.bind(addressIListenFrom, localPort) .then((RawDatagramSocket socket) { + // 设置接收缓冲区大小 (SO_RCVBUF = 8) + socket.setRawOption( + RawSocketOption.fromInt( + RawSocketOption.levelSocket, + 8, // SO_RCVBUF for Android/iOS + 2 * 1024 * 1024, // 2MB receive buffer + ), + ); + + // 设置发送缓冲区大小 (SO_SNDBUF = 7) + socket.setRawOption( + RawSocketOption.fromInt( + RawSocketOption.levelSocket, + 7, // SO_SNDBUF for Android/iOS + 2 * 1024 * 1024, // 2MB send buffer + ), + ); + _udpSocket = socket; /// 广播功能 @@ -1017,35 +1051,25 @@ class StartChartManage { void _onReceiveData(RawDatagramSocket socket, BuildContext context) { socket.listen((RawSocketEvent event) { if (event == RawSocketEvent.read) { - Datagram? dg = socket.receive(); - try { - if (dg?.data != null) { - final deserialize = ScpMessage.deserialize(dg!.data); - - // //ToDo: 增加对讲调试、正式可删除 - // UdpTalkDataHandler().updateRecvDataRate(dg.data.length); - - // // 更新调试信息 - // Provider.of(context, listen: false).updateDebugInfo( - // UdpTalkDataHandler().getLastRecvDataRate() ~/ 1024, // 转换为KB - // UdpTalkDataHandler().getLastRecvPacketCount(), - // UdpTalkDataHandler().getLastSendDataRate() ~/ 1024, // 转换为KB - // UdpTalkDataHandler().getLastSendPacketCount(), - // ); - - if (deserialize != null) { - // 处理返回数据 - _handleUdpResultData(deserialize); - } - if (deserialize.PayloadType != PayloadTypeConstant.heartbeat) { - if (deserialize.Payload != null) { - // _log(text: 'Udp收到结构体数据---》$deserialize'); + Datagram? dg; + while ((dg = socket.receive()) != null) { + try { + if (dg?.data != null) { + // Fallback: 主线程处理（极端情况） + // 更好的做法：批量处理 + final deserialize = ScpMessage.deserialize(dg!.data); + // if (deserialize.PayloadType == PayloadTypeConstant.talkData) { + // _log( + // text: 'mesaageId:${deserialize.MessageId},' + // 'SpTotal:${deserialize.SpTotal},SpIndex:${deserialize.SpIndex}'); + // } + if (deserialize != null) { + _handleUdpResultData(deserialize); } - // _log(text: 'text---》${utf8.decode(deserialize.Payload)}'); } + } catch (e, stackTrace) { + throw StartChartMessageException('$e\n,$stackTrace'); } - } catch (e, stackTrace) { - throw StartChartMessageException('$e\n,$stackTrace'); } } }); @@ -1056,14 +1080,6 @@ class StartChartManage { final int payloadType = scpMessage.PayloadType ?? 0; final int messageType = scpMessage.MessageType ?? 0; try { - // 记录分包数据用于统计丢包率 - if (scpMessage.SpIndex != null && - scpMessage.SpTotal != null && - scpMessage.MessageId != null) { - PacketLossStatistics().recordPacket( - scpMessage.MessageId!, scpMessage.SpIndex!, scpMessage.SpTotal!); - } - final ScpMessageHandler handler = ScpMessageHandlerFactory.createHandler(payloadType); if (messageType == MessageTypeConstant.Req) { diff --git a/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart b/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart index 9ddf4a57..97522643 100644 --- a/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart +++ b/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart @@ -25,6 +25,7 @@ import 'package:star_lock/network/api_repository.dart'; import 'package:star_lock/talk/call/callTalk.dart'; import 'package:star_lock/talk/call/g711.dart'; import 'package:star_lock/talk/starChart/constant/talk_status.dart'; +import 'package:star_lock/talk/starChart/entity/scp_message.dart'; import 'package:star_lock/talk/starChart/handle/other/packet_loss_statistics.dart'; import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; @@ -37,6 +38,7 @@ import 'package:star_lock/tools/G711Tool.dart'; import 'package:star_lock/tools/bugly/bugly_tool.dart'; import 'package:star_lock/tools/commonDataManage.dart'; import 'package:star_lock/tools/storage.dart'; +import 'package:video_decode_plugin/nalu_utils.dart'; import 'package:video_decode_plugin/video_decode_plugin.dart'; import '../../../../tools/baseGetXController.dart'; @@ -50,6 +52,15 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { int audioBufferSize = 2; // 音频默认缓冲2帧 + // 回绕阈值，动态调整，frameSeq较小时阈值也小 + int _getFrameSeqRolloverThreshold(int lastSeq) { + if (lastSeq > 2000) { + return 1000; + } else { + return (lastSeq / 2).round(); + } + } + // 定义音频帧缓冲和发送函数 final List _bufferedAudioFrames = []; @@ -87,15 +98,17 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { // 新增：等待新I帧状态 bool _waitingForIFrame = false; + int? lastDecodedIFrameSeq; + // 初始化视频解码器 Future _initVideoDecoder() async { try { state.isLoading.value = true; // 创建解码器配置 final config = VideoDecoderConfig( - width: 864, + width: StartChartManage().videoWidth, // 实际视频宽度 - height: 480, + height: StartChartManage().videoHeight, codecType: 'h264', ); // 初始化解码器并获取textureId @@ -157,29 +170,37 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { int pts, int frameSeq, int frameSeqI, + ScpMessage scpMessage, ) { - // 检测frameSeq回绕，且为I帧 + + // 动态回绕阈值判断，frameSeq较小时阈值也小 if (!_pendingStreamReset && _lastFrameSeq != null && frameType == TalkDataH264Frame_FrameTypeE.I && frameSeq < _lastFrameSeq!) { - // 检测到新流I帧，进入loading并重置所有本地状态 - AppLog.log( - '检测到新流I帧，frameSeq回绕，进入loading并重置: frameSeq=$frameSeq, lastFrameSeq=$_lastFrameSeq'); - Future.microtask(() => state.isLoading.value = true); - _pendingStreamReset = true; - // 先暂停帧处理定时器，防止竞态 - _stopFrameProcessTimer(); - // 先释放并重新初始化解码器 - _resetDecoderForNewStream(_pendingResetWidth, _pendingResetHeight); - // 重置所有本地状态 - _lastFrameSeq = null; - _decodedIFrames.clear(); - state.h264FrameBuffer.clear(); - // 再恢复帧处理定时器 - _startFrameProcessTimer(); - // 不return，直接用该I帧初始化解码器并解码 - // 继续往下执行 + int dynamicThreshold = _getFrameSeqRolloverThreshold(_lastFrameSeq!); + if ((_lastFrameSeq! - frameSeq) > dynamicThreshold) { + // 检测到新流I帧，frameSeq大幅回绕，进入loading并重置所有本地状态 + AppLog.log('检测到新流I帧，frameSeq大幅回绕，进入loading并重置: frameSeq=$frameSeq, lastFrameSeq=$_lastFrameSeq, 阈值=$dynamicThreshold'); + Future.microtask(() => state.isLoading.value = true); + _pendingStreamReset = true; + // 先暂停帧处理定时器，防止竞态 + _stopFrameProcessTimer(); + // 先释放并重新初始化解码器 + _resetDecoderForNewStream(_pendingResetWidth, _pendingResetHeight); + // 重置所有本地状态 + _lastFrameSeq = null; + _decodedIFrames.clear(); + state.h264FrameBuffer.clear(); + // 再恢复帧处理定时器 + _startFrameProcessTimer(); + // 不return，直接用该I帧初始化解码器并解码 + // 继续往下执行 + } else { + // 小幅度乱序，直接丢弃 + AppLog.log('检测到I帧乱序（未超过回绕阈值$dynamicThreshold），丢弃: frameSeq=$frameSeq, lastFrameSeq=$_lastFrameSeq'); + return; + } } // 如果处于pendingStreamReset，等待新I帧 if (_pendingStreamReset) { @@ -212,6 +233,7 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { 'frameSeq': frameSeq, 'frameSeqI': frameSeqI, 'pts': pts, + 'scpMessage': scpMessage, }; // 如果缓冲区超出最大大小，优先丢弃P/B帧 @@ -257,14 +279,25 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { return; } - // 设置正在处理标志 - state.isProcessingFrame = true; + // 优先查找I帧，按frameSeq最小的I帧消费 + final iFrames = state.h264FrameBuffer + .where((f) => f['frameType'] == TalkDataH264Frame_FrameTypeE.I) + .toList(); + iFrames + .sort((a, b) => (a['frameSeq'] as int).compareTo(b['frameSeq'] as int)); - try { - // 取出最早的帧 - final Map? frameMap = state.h264FrameBuffer.isNotEmpty - ? state.h264FrameBuffer.removeAt(0) - : null; + if (iFrames.isNotEmpty) { + // 有I帧，消费最小的I帧，并记录其frameSeq + final minIFrame = iFrames.first; + final minIFrameSeq = minIFrame['frameSeq']; + final targetIndex = state.h264FrameBuffer.indexWhere( + (f) => + f['frameType'] == TalkDataH264Frame_FrameTypeE.I && + f['frameSeq'] == minIFrameSeq, + ); + state.isProcessingFrame = true; + final Map? frameMap = + state.h264FrameBuffer.removeAt(targetIndex); if (frameMap == null) { state.isProcessingFrame = false; return; @@ -274,6 +307,7 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { final int? frameSeq = frameMap['frameSeq']; final int? frameSeqI = frameMap['frameSeqI']; final int? pts = frameMap['pts']; + final ScpMessage? scpMessage = frameMap['scpMessage']; if (frameData == null || frameType == null || frameSeq == null || @@ -282,25 +316,86 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { state.isProcessingFrame = false; return; } - // 解码器未初始化或textureId为null时跳过 if (state.textureId.value == null) { state.isProcessingFrame = false; return; } + lastDecodedIFrameSeq = minIFrameSeq; + // AppLog.log('送入解码器的P帧数据frameSeq:${frameSeq},frameSeqI:${frameSeqI},' + // 'frameType:${frameType},messageId:${scpMessage!.MessageId}'); + // final spsData = NaluUtils.filterNalusByType(frameData, 7); + // final ppsData = NaluUtils.filterNalusByType(frameData, 8); + // AppLog.log('SPSDATA:${spsData}，ppsData:${ppsData}'); await VideoDecodePlugin.sendFrame( frameData: frameData, - frameType: frameType == TalkDataH264Frame_FrameTypeE.I ? 0 : 1, + frameType: 0, frameSeq: frameSeq, timestamp: pts, splitNalFromIFrame: true, refIFrameSeq: frameSeqI, ); - } catch (e) { - AppLog.log('处理缓冲帧失败: $e'); - } finally { - // 重置处理标志 state.isProcessingFrame = false; + return; } + + // 没有I帧时，只消费refIFrameSeq等于lastDecodedIFrameSeq的P帧 + if (lastDecodedIFrameSeq != null) { + final validPFrames = state.h264FrameBuffer + .where((f) => + f['frameType'] == TalkDataH264Frame_FrameTypeE.P && + f['frameSeqI'] == lastDecodedIFrameSeq) + .toList(); + if (validPFrames.isNotEmpty) { + validPFrames.sort( + (a, b) => (a['frameSeq'] as int).compareTo(b['frameSeq'] as int)); + final minPFrame = validPFrames.first; + final targetIndex = state.h264FrameBuffer.indexWhere( + (f) => + f['frameType'] == TalkDataH264Frame_FrameTypeE.P && + f['frameSeq'] == minPFrame['frameSeq'] && + f['frameSeqI'] == lastDecodedIFrameSeq, + ); + state.isProcessingFrame = true; + final Map? frameMap = + state.h264FrameBuffer.removeAt(targetIndex); + if (frameMap == null) { + state.isProcessingFrame = false; + return; + } + final List? frameData = frameMap['frameData']; + final TalkDataH264Frame_FrameTypeE? frameType = frameMap['frameType']; + final int? frameSeq = frameMap['frameSeq']; + final int? frameSeqI = frameMap['frameSeqI']; + final int? pts = frameMap['pts']; + final ScpMessage? scpMessage = frameMap['scpMessage']; + if (frameData == null || + frameType == null || + frameSeq == null || + frameSeqI == null || + pts == null) { + state.isProcessingFrame = false; + return; + } + if (state.textureId.value == null) { + state.isProcessingFrame = false; + return; + } + // AppLog.log('送入解码器的I帧数据frameSeq:${frameSeq},frameSeqI:${frameSeqI},' + // 'frameType:${frameType},messageId:${scpMessage!.MessageId}'); + + await VideoDecodePlugin.sendFrame( + frameData: frameData, + frameType: 1, + frameSeq: frameSeq, + timestamp: pts, + splitNalFromIFrame: true, + refIFrameSeq: frameSeqI, + ); + state.isProcessingFrame = false; + return; + } + } + // 其他情况不消费，等待I帧到来 } /// 停止帧处理定时器 @@ -318,6 +413,11 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { // 监听音视频数据流 void _startListenTalkData() { + // 取消旧订阅 + if (_streamSubscription != null) { + _streamSubscription!.cancel(); + _streamSubscription = null; + } // 防止重复监听 if (_isListening) { AppLog.log("已经存在数据流监听，避免重复监听"); @@ -329,37 +429,7 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { _streamSubscription = state.talkDataRepository.talkDataStream .listen((TalkDataModel talkDataModel) async { - final talkData = talkDataModel.talkData; - final talkDataH264Frame = talkDataModel.talkDataH264Frame; - final contentType = talkData!.contentType; - - // 判断数据类型，进行分发处理 - switch (contentType) { - case TalkData_ContentTypeE.G711: - if (state.audioBuffer.length >= audioBufferSize) { - state.audioBuffer.removeAt(0); // 丢弃最旧的数据 - } - state.audioBuffer.add(talkData); // 添加新数据 - // 添加音频播放逻辑，与视频类似 - _playAudioFrames(); - break; - case TalkData_ContentTypeE.H264: - // 处理H264帧 - if (state.textureId.value != null) { - if (talkDataH264Frame != null) { - _addFrameToBuffer( - talkData.content, - talkDataH264Frame.frameType, - talkData.durationMs, - talkDataH264Frame.frameSeq, - talkDataH264Frame.frameSeqI, - ); - } - } else { - AppLog.log('无法处理H264帧：textureId为空'); - } - break; - } + _processFrame(talkDataModel); }); } @@ -585,7 +655,8 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { TalkExpectReq talkExpectReq = TalkExpectReq(); state.isOpenVoice.value = !state.isOpenVoice.value; // 根据当前清晰度动态设置videoType - VideoTypeE currentVideoType = qualityToVideoType[state.currentQuality.value] ?? VideoTypeE.H264; + VideoTypeE currentVideoType = + qualityToVideoType[state.currentQuality.value] ?? VideoTypeE.H264; if (!state.isOpenVoice.value) { talkExpectReq = TalkExpectReq( videoType: [currentVideoType], @@ -1104,144 +1175,144 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { } // 新增：I帧处理方法 - void _handleIFrameWithSpsPpsAndIdr( - List frameData, int durationMs, int frameSeq, int frameSeqI) { - // 清空缓冲区，丢弃I帧前所有未处理帧（只保留SPS/PPS/I帧） - state.h264FrameBuffer.clear(); - _extractAndBufferSpsPpsForBuffer( - frameData, durationMs, frameSeq, frameSeqI); - // 只要缓存有SPS/PPS就先写入，再写I帧本体（只写IDR） - if (spsCache == null || ppsCache == null) { - // 没有SPS/PPS缓存，丢弃本次I帧 - return; - } - // 先写入SPS/PPS - _addFrameToBuffer(spsCache!, TalkDataH264Frame_FrameTypeE.I, durationMs, - frameSeq, frameSeqI); - _addFrameToBuffer(ppsCache!, TalkDataH264Frame_FrameTypeE.I, durationMs, - frameSeq, frameSeqI); - // 分割I帧包，只写入IDR（type 5） - List> nalus = []; - int i = 0; - List data = frameData; - while (i < data.length - 3) { - int start = -1; - int next = -1; - if (data[i] == 0x00 && data[i + 1] == 0x00) { - if (data[i + 2] == 0x01) { - start = i; - i += 3; - } else if (i + 3 < data.length && - data[i + 2] == 0x00 && - data[i + 3] == 0x01) { - start = i; - i += 4; - } else { - i++; - continue; - } - next = i; - while (next < data.length - 3) { - if (data[next] == 0x00 && - data[next + 1] == 0x00 && - ((data[next + 2] == 0x01) || - (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { - break; - } - next++; - } - nalus.add(data.sublist(start, next)); - i = next; - } else { - i++; - } - } - int nalusTotalLen = - nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; - if (nalus.isEmpty && data.isNotEmpty) { - nalus.add(data); - } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { - nalus.add(data.sublist(nalusTotalLen)); - } - for (final nalu in nalus) { - int offset = 0; - if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { - if (nalu[2] == 0x01) - offset = 3; - else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; - } - if (nalu.length > offset) { - int naluType = nalu[offset] & 0x1F; - if (naluType == 5) { - _addFrameToBuffer(nalu, TalkDataH264Frame_FrameTypeE.I, durationMs, - frameSeq, frameSeqI); - } - } - } - } + // void _handleIFrameWithSpsPpsAndIdr( + // List frameData, int durationMs, int frameSeq, int frameSeqI) { + // // 清空缓冲区，丢弃I帧前所有未处理帧（只保留SPS/PPS/I帧） + // state.h264FrameBuffer.clear(); + // _extractAndBufferSpsPpsForBuffer( + // frameData, durationMs, frameSeq, frameSeqI); + // // 只要缓存有SPS/PPS就先写入，再写I帧本体（只写IDR） + // if (spsCache == null || ppsCache == null) { + // // 没有SPS/PPS缓存，丢弃本次I帧 + // return; + // } + // // 先写入SPS/PPS + // _addFrameToBuffer(spsCache!, TalkDataH264Frame_FrameTypeE.I, durationMs, + // frameSeq, frameSeqI); + // _addFrameToBuffer(ppsCache!, TalkDataH264Frame_FrameTypeE.I, durationMs, + // frameSeq, frameSeqI); + // // 分割I帧包，只写入IDR（type 5） + // List> nalus = []; + // int i = 0; + // List data = frameData; + // while (i < data.length - 3) { + // int start = -1; + // int next = -1; + // if (data[i] == 0x00 && data[i + 1] == 0x00) { + // if (data[i + 2] == 0x01) { + // start = i; + // i += 3; + // } else if (i + 3 < data.length && + // data[i + 2] == 0x00 && + // data[i + 3] == 0x01) { + // start = i; + // i += 4; + // } else { + // i++; + // continue; + // } + // next = i; + // while (next < data.length - 3) { + // if (data[next] == 0x00 && + // data[next + 1] == 0x00 && + // ((data[next + 2] == 0x01) || + // (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { + // break; + // } + // next++; + // } + // nalus.add(data.sublist(start, next)); + // i = next; + // } else { + // i++; + // } + // } + // int nalusTotalLen = + // nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; + // if (nalus.isEmpty && data.isNotEmpty) { + // nalus.add(data); + // } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { + // nalus.add(data.sublist(nalusTotalLen)); + // } + // for (final nalu in nalus) { + // int offset = 0; + // if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { + // if (nalu[2] == 0x01) + // offset = 3; + // else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; + // } + // if (nalu.length > offset) { + // int naluType = nalu[offset] & 0x1F; + // if (naluType == 5) { + // _addFrameToBuffer(nalu, TalkDataH264Frame_FrameTypeE.I, durationMs, + // frameSeq, frameSeqI); + // } + // } + // } + // } // 新增：P帧处理方法 - void _handlePFrame( - List frameData, int durationMs, int frameSeq, int frameSeqI) { - // 只写入P帧（type 1） - List> nalus = []; - int i = 0; - List data = frameData; - while (i < data.length - 3) { - int start = -1; - int next = -1; - if (data[i] == 0x00 && data[i + 1] == 0x00) { - if (data[i + 2] == 0x01) { - start = i; - i += 3; - } else if (i + 3 < data.length && - data[i + 2] == 0x00 && - data[i + 3] == 0x01) { - start = i; - i += 4; - } else { - i++; - continue; - } - next = i; - while (next < data.length - 3) { - if (data[next] == 0x00 && - data[next + 1] == 0x00 && - ((data[next + 2] == 0x01) || - (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { - break; - } - next++; - } - nalus.add(data.sublist(start, next)); - i = next; - } else { - i++; - } - } - int nalusTotalLen = - nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; - if (nalus.isEmpty && data.isNotEmpty) { - nalus.add(data); - } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { - nalus.add(data.sublist(nalusTotalLen)); - } - for (final nalu in nalus) { - int offset = 0; - if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { - if (nalu[2] == 0x01) - offset = 3; - else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; - } - if (nalu.length > offset) { - int naluType = nalu[offset] & 0x1F; - if (naluType == 1) { - _addFrameToBuffer(nalu, TalkDataH264Frame_FrameTypeE.P, durationMs, - frameSeq, frameSeqI); - } - } - } - } + // void _handlePFrame( + // List frameData, int durationMs, int frameSeq, int frameSeqI) { + // // 只写入P帧（type 1） + // List> nalus = []; + // int i = 0; + // List data = frameData; + // while (i < data.length - 3) { + // int start = -1; + // int next = -1; + // if (data[i] == 0x00 && data[i + 1] == 0x00) { + // if (data[i + 2] == 0x01) { + // start = i; + // i += 3; + // } else if (i + 3 < data.length && + // data[i + 2] == 0x00 && + // data[i + 3] == 0x01) { + // start = i; + // i += 4; + // } else { + // i++; + // continue; + // } + // next = i; + // while (next < data.length - 3) { + // if (data[next] == 0x00 && + // data[next + 1] == 0x00 && + // ((data[next + 2] == 0x01) || + // (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { + // break; + // } + // next++; + // } + // nalus.add(data.sublist(start, next)); + // i = next; + // } else { + // i++; + // } + // } + // int nalusTotalLen = + // nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; + // if (nalus.isEmpty && data.isNotEmpty) { + // nalus.add(data); + // } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { + // nalus.add(data.sublist(nalusTotalLen)); + // } + // for (final nalu in nalus) { + // int offset = 0; + // if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { + // if (nalu[2] == 0x01) + // offset = 3; + // else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; + // } + // if (nalu.length > offset) { + // int naluType = nalu[offset] & 0x1F; + // if (naluType == 1) { + // _addFrameToBuffer(nalu, TalkDataH264Frame_FrameTypeE.P, durationMs, + // frameSeq, frameSeqI); + // } + // } + // } + // } // 切换清晰度的方法，后续补充具体实现 void onQualityChanged(String quality) async { @@ -1293,30 +1364,92 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { // 新增：重置解码器方法 Future _resetDecoderForNewStream(int width, int height) async { try { + // 先停止帧处理定时器 + _stopFrameProcessTimer(); + + // 释放旧解码器 if (state.textureId.value != null) { await VideoDecodePlugin.releaseDecoder(); - Future.microtask(() => state.textureId.value = null); + state.textureId.value = null; } + + // 等待一小段时间确保资源释放完成 + await Future.delayed(Duration(milliseconds: 100)); + + // 创建新的解码器配置 final config = VideoDecoderConfig( width: width, height: height, codecType: 'h264', ); + + // 初始化新解码器 final textureId = await VideoDecodePlugin.initDecoder(config); if (textureId != null) { - Future.microtask(() => state.textureId.value = textureId); + state.textureId.value = textureId; AppLog.log('frameSeq回绕后解码器初始化成功：textureId=$textureId'); + + // 设置帧渲染监听 VideoDecodePlugin.setOnFrameRenderedListener((textureId) { AppLog.log('已经开始渲染======='); // 只有真正渲染出首帧时才关闭loading - Future.microtask(() => state.isLoading.value = false); + state.isLoading.value = false; }); + + // 重新启动帧处理定时器 + _startFrameProcessTimer(); + + // 重置相关状态 + _decodedIFrames.clear(); + state.h264FrameBuffer.clear(); + state.isProcessingFrame = false; + hasSps = false; + hasPps = false; + spsCache = null; + ppsCache = null; } else { AppLog.log('frameSeq回绕后解码器初始化失败'); + state.isLoading.value = false; } - _startFrameProcessTimer(); } catch (e) { AppLog.log('frameSeq回绕时解码器初始化错误: $e'); + state.isLoading.value = false; + } + } + + void _processFrame(TalkDataModel talkDataModel) { + final talkData = talkDataModel.talkData; + final talkDataH264Frame = talkDataModel.talkDataH264Frame; + final scpMessage = talkDataModel.scpMessage; + final contentType = talkData!.contentType; + + // 判断数据类型，进行分发处理 + switch (contentType) { + case TalkData_ContentTypeE.G711: + if (state.audioBuffer.length >= audioBufferSize) { + state.audioBuffer.removeAt(0); // 丢弃最旧的数据 + } + state.audioBuffer.add(talkData); // 添加新数据 + // 添加音频播放逻辑，与视频类似 + _playAudioFrames(); + break; + case TalkData_ContentTypeE.H264: + // 处理H264帧 + if (state.textureId.value != null) { + if (talkDataH264Frame != null) { + _addFrameToBuffer( + talkData.content, + talkDataH264Frame.frameType, + talkData.durationMs, + talkDataH264Frame.frameSeq, + talkDataH264Frame.frameSeqI, + scpMessage!, + ); + } + } else { + AppLog.log('无法处理H264帧：textureId为空'); + } + break; } } } diff --git a/lib/talk/starChart/views/native/talk_view_native_decode_page.dart b/lib/talk/starChart/views/native/talk_view_native_decode_page.dart index 84c29c3d..eeeea7cf 100644 --- a/lib/talk/starChart/views/native/talk_view_native_decode_page.dart +++ b/lib/talk/starChart/views/native/talk_view_native_decode_page.dart @@ -114,16 +114,16 @@ class _TalkViewNativeDecodePageState extends State key: state.globalKey, child: SizedBox.expand( child: RotatedBox( - // 解码器不支持硬件旋转，使用RotatedBox - quarterTurns: startChartManage.rotateAngle ~/ 90, - child: Platform.isIOS - ? Transform.scale( - scale: 1.008, // 轻微放大，消除iOS白边 - child: Texture( - textureId: state.textureId.value!, - filterQuality: FilterQuality.medium, - ), - ) + // 解码器不支持硬件旋转，使用RotatedBox + quarterTurns: startChartManage.rotateAngle ~/ 90, + child: Platform.isIOS + ? Transform.scale( + scale: 1.008, // 轻微放大，消除iOS白边 + child: Texture( + textureId: state.textureId.value!, + filterQuality: FilterQuality.medium, + ), + ) : Texture( textureId: state.textureId.value!, filterQuality: FilterQuality.medium, diff --git a/lib/talk/starChart/views/native/talk_view_native_decode_state.dart b/lib/talk/starChart/views/native/talk_view_native_decode_state.dart index c9be5f57..e3bf6908 100644 --- a/lib/talk/starChart/views/native/talk_view_native_decode_state.dart +++ b/lib/talk/starChart/views/native/talk_view_native_decode_state.dart @@ -110,8 +110,8 @@ class TalkViewNativeDecodeState { // H264帧缓冲区相关 final List> h264FrameBuffer = >[]; // H264帧缓冲区，存储帧数据和类型 - final int maxFrameBufferSize = 15; // 最大缓冲区大小 - final int targetFps = 30; // 目标解码帧率,只是为了快速填充native的缓冲区 + final int maxFrameBufferSize = 50; // 最大缓冲区大小 + final int targetFps = 60; // 目标解码帧率,只是为了快速填充native的缓冲区 Timer? frameProcessTimer; // 帧处理定时器 bool isProcessingFrame = false; // 是否正在处理帧 int lastProcessedTimestamp = 0; // 上次处理帧的时间戳 diff --git a/pubspec.yaml b/pubspec.yaml index 3ddc74b4..ae04ec9f 100755 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -130,7 +130,7 @@ dependencies: video_decode_plugin: git: url: git@code.star-lock.cn:liyi/video_decode_plugin.git - ref: 68bb4b7fb637ef5a78856908e1bc464f50fe967a + ref: 5dfbd190fdc61dab3fc93543606b85d6b826a2ed flutter_localizations: sdk: flutter @@ -319,6 +319,7 @@ flutter: assets: - images/ - images/tabbar/ + - images/guide/ - images/main/ - images/mine/ - images/lockType/