From 1784f75c475dde791943022bc67678cac9997b85 Mon Sep 17 00:00:00 2001 From: liyi Date: Fri, 25 Apr 2025 10:21:05 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E5=A2=9E=E5=8A=A0=E5=8E=9F=E7=94=9F?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E8=A7=A3=E7=A0=81=E7=9A=84=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E3=80=81=E5=A2=9E=E5=8A=A0h264=E3=80=81mjpeg=E5=88=87=E6=8D=A2?= =?UTF-8?q?=E7=9A=84debug=E6=8C=89=E9=92=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/appRouters.dart | 2 + .../lockDetail/lockDetail_logic.dart | 19 +- .../lockDetail/lockDetail_page.dart | 58 +- .../lockDetail/lockDetail_state.dart | 11 +- lib/network/api_provider.dart | 2 +- lib/network/api_repository.dart | 2 +- .../handle/impl/udp_talk_data_handler.dart | 3 - .../handle/other/h264_frame_handler.dart | 3 - .../handle/other/packet_loss_statistics.dart | 84 +- lib/talk/starChart/star_chart_manage.dart | 2 +- .../native/talk_view_native_decode_logic.dart | 798 ++++++++++++++++++ .../native/talk_view_native_decode_page.dart | 557 ++++++++++++ .../native/talk_view_native_decode_state.dart | 109 +++ .../views/talkView/talk_view_logic.dart | 37 +- .../starChart/webView/h264_web_logic.dart | 20 +- pubspec.yaml | 3 +- 16 files changed, 1672 insertions(+), 38 deletions(-) create mode 100644 lib/talk/starChart/views/native/talk_view_native_decode_logic.dart create mode 100644 lib/talk/starChart/views/native/talk_view_native_decode_page.dart create mode 100644 lib/talk/starChart/views/native/talk_view_native_decode_state.dart diff --git a/lib/appRouters.dart b/lib/appRouters.dart index ab6c17a9..83b41650 100755 --- a/lib/appRouters.dart +++ b/lib/appRouters.dart @@ -60,6 +60,7 @@ import 'package:star_lock/mine/mineSet/transferSmartLock/transferSmartLockList/t import 'package:star_lock/mine/valueAddedServices/advancedFeaturesWeb/advancedFeaturesWeb_page.dart'; import 'package:star_lock/mine/valueAddedServices/advancedFunctionRecord/advancedFunctionRecord_page.dart'; import 'package:star_lock/mine/valueAddedServices/valueAddedServicesRecord/value_added_services_record_page.dart'; +import 'package:star_lock/talk/starChart/views/native/talk_view_native_decode_page.dart'; import 'package:star_lock/talk/starChart/views/talkView/talk_view_page.dart'; import 'package:star_lock/talk/starChart/webView/h264_web_view.dart'; @@ -1184,6 +1185,7 @@ abstract class AppRouters { page: () => const DoubleLockLinkPage()), GetPage( name: Routers.starChartTalkView, page: () => const TalkViewPage()), + // GetPage(name: Routers.h264WebView, page: () => TalkViewNativeDecodePage()), GetPage(name: Routers.h264WebView, page: () => H264WebView()), ]; } diff --git a/lib/main/lockDetail/lockDetail/lockDetail_logic.dart b/lib/main/lockDetail/lockDetail/lockDetail_logic.dart index c03a2753..72683016 100755 --- a/lib/main/lockDetail/lockDetail/lockDetail_logic.dart +++ b/lib/main/lockDetail/lockDetail/lockDetail_logic.dart @@ -15,6 +15,8 @@ import 'package:star_lock/main/lockDetail/lockDetail/device_network_info.dart'; import 'package:star_lock/main/lockDetail/lockSet/lockTime/getServerDatetime_entity.dart'; import 'package:star_lock/main/lockMian/entity/lockListInfo_entity.dart'; import 'package:star_lock/talk/starChart/constant/talk_status.dart'; +import 'package:star_lock/talk/starChart/handle/other/packet_loss_statistics.dart'; +import 'package:star_lock/talk/starChart/proto/talk_expect.pb.dart'; import 'package:star_lock/talk/starChart/star_chart_manage.dart'; import 'package:star_lock/tools/bugly/bugly_tool.dart'; import 'package:star_lock/tools/throttler.dart'; @@ -564,7 +566,7 @@ class LockDetailLogic extends BaseGetXController { // 获取手机联网token，根据锁设置里面获取的开锁时是否联网来判断是否调用这个接口 Future getLockNetToken() async { final LockNetTokenEntity entity = await ApiRepository.to - .getLockNetToken(lockId: state.keyInfos.value.lockId.toString()); + .getLockNetToken(lockId: state.keyInfos.value.lockId!); if (entity.errorCode!.codeIsSuccessful) { state.lockNetToken = entity.data!.token!.toString(); // AppLog.log('从服务器获取联网token:${state.lockNetToken}'); @@ -769,12 +771,12 @@ class LockDetailLogic extends BaseGetXController { if (catEyeConfig.isNotEmpty && catEyeConfig.length > 0 && catEyeConfig[0].catEyeMode != 0) { - if (StartChartManage().lockNetworkInfo.wifiName == null || - StartChartManage().lockNetworkInfo.wifiName == '') { + if ((StartChartManage().lockNetworkInfo.wifiName == null || + StartChartManage().lockNetworkInfo.wifiName == '') ) { showToast('设备未配网'.tr); return; } - + PacketLossStatistics().reset(); // 发送监控id StartChartManage().startCallRequestMessageTimer( ToPeerId: StartChartManage().lockNetworkInfo.peerId ?? ''); @@ -795,6 +797,15 @@ class LockDetailLogic extends BaseGetXController { @override void onInit() { super.onInit(); + + // 初始化开关状态为当前对讲视频模式 + final currentTalkExpect = StartChartManage().getDefaultTalkExpect(); + if (currentTalkExpect.videoType.contains(VideoTypeE.H264)) { + state.useH264Mode.value = true; + } else if (currentTalkExpect.videoType.contains(VideoTypeE.IMAGE)) { + state.useH264Mode.value = false; + } + state.LockSetChangeSetRefreshLockDetailWithTypeSubscription = eventBus .on() .listen((LockSetChangeSetRefreshLockDetailWithType event) { diff --git a/lib/main/lockDetail/lockDetail/lockDetail_page.dart b/lib/main/lockDetail/lockDetail/lockDetail_page.dart index 1040b5ba..93ea8f2f 100755 --- a/lib/main/lockDetail/lockDetail/lockDetail_page.dart +++ b/lib/main/lockDetail/lockDetail/lockDetail_page.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; import 'package:flutter_easyloading/flutter_easyloading.dart'; import 'package:flutter_screenutil/flutter_screenutil.dart'; @@ -88,7 +89,6 @@ class _LockDetailPageState extends State /// 路由订阅 AppRouteObserver().routeObserver.subscribe(this, ModalRoute.of(context)!); state.isOpenLockNeedOnline.refresh(); - } StreamSubscription? _lockRefreshLockDetailInfoDataEvent; @@ -507,6 +507,60 @@ class _LockDetailPageState extends State Widget skWidget() { return ListView( children: [ + // Container( + // padding: EdgeInsets.symmetric(vertical: 15, horizontal: 20), + // margin: EdgeInsets.only(top: 10, bottom: 10), + // decoration: BoxDecoration( + // color: Colors.white, + // borderRadius: BorderRadius.circular(10), + // boxShadow: [ + // BoxShadow( + // color: Colors.black.withOpacity(0.05), + // blurRadius: 5, + // offset: Offset(0, 2), + // ), + // ], + // ), + // child: Row( + // mainAxisAlignment: MainAxisAlignment.spaceBetween, + // children: [ + // Text('对讲视频模式'.tr, + // style: TextStyle(fontSize: 16, fontWeight: FontWeight.bold)), + // Row( + // children: [ + // Text('mjpeg', + // style: TextStyle( + // fontSize: 14, + // color: !state.useH264Mode.value + // ? AppColors.mainColor + // : Colors.grey)), + // Obx(() => Switch( + // value: state.useH264Mode.value, + // activeColor: AppColors.mainColor, + // onChanged: (value) { + // state.useH264Mode.value = value; + // if (value) { + // // 使用H264模式 + // StartChartManage() + // .sendH264VideoAndG711AudioTalkExpectData(); + // } else { + // // 使用Image模式 + // StartChartManage() + // .sendImageVideoAndG711AudioTalkExpectData(); + // } + // }, + // )), + // Text('H264'.tr, + // style: TextStyle( + // fontSize: 14, + // color: state.useH264Mode.value + // ? AppColors.mainColor + // : Colors.grey)), + // ], + // ), + // ], + // ), + // ), Visibility( visible: (state.keyInfos.value.keyType == XSConstantMacro.keyTypeTime || @@ -1467,7 +1521,7 @@ class _LockDetailPageState extends State state.iSOpenLock.value = true; state.openLockBtnState.value = 1; state.animationController!.forward(); - // AppLog.log('点击开锁'); + AppLog.log('点击开锁'); if (isOpenLockNeedOnline) { // 不需要联网 state.openDoorModel = 0; diff --git a/lib/main/lockDetail/lockDetail/lockDetail_state.dart b/lib/main/lockDetail/lockDetail/lockDetail_state.dart index 01178f52..a949b693 100755 --- a/lib/main/lockDetail/lockDetail/lockDetail_state.dart +++ b/lib/main/lockDetail/lockDetail/lockDetail_state.dart @@ -7,18 +7,18 @@ import 'package:star_lock/main/lockDetail/lockSet/lockSet/lockSetInfo_entity.dar import '../../../blue/io_reply.dart'; import '../../lockMian/entity/lockListInfo_entity.dart'; - class LockDetailState { Rx keyInfos = LockListInfoItemEntity().obs; final Rx lockSetInfoData = LockSetInfoData().obs; late StreamSubscription replySubscription; - StreamSubscription? lockSetOpenOrCloseCheckInRefreshLockDetailWithAttendanceEvent; + StreamSubscription? + lockSetOpenOrCloseCheckInRefreshLockDetailWithAttendanceEvent; StreamSubscription? LockSetChangeSetRefreshLockDetailWithTypeSubscription; StreamSubscription? DetailLockInfo; StreamSubscription? SuccessfulDistributionNetworkEvent; String lockNetToken = '0'; - int differentialTime = 0;// 服务器时间与本地时间差值 + int differentialTime = 0; // 服务器时间与本地时间差值 bool isHaveNetwork = true; int lockUserNo = 0; int senderUserId = 0; @@ -41,7 +41,7 @@ class LockDetailState { RxBool bottomBtnisEable = true.obs; // 是否不可用 用于限制底部按钮是否可用 RxBool openDoorBtnisUneable = true.obs; // 当钥匙状态不能使用的情况下开锁按钮禁止使用,默认可用 - int openDoorModel = 0;// 离线开门0, 在线开门2 离线关门32 在线关门34 + int openDoorModel = 0; // 离线开门0, 在线开门2 离线关门32 在线关门34 //过渡动画控制器 AnimationController? animationController; @@ -58,6 +58,9 @@ class LockDetailState { int logCountPage = 10; // 蓝牙记录一页多少个 RxInt nextAuthTime = 0.obs; // 下次认证时间 + // 视频编码模式选择开关状态 + RxBool useH264Mode = true.obs; // true表示使用H264模式，false表示使用Image模式 + // LockDetailState() { // Map map = Get.arguments; // lockCount = map["lockCount"]; diff --git a/lib/network/api_provider.dart b/lib/network/api_provider.dart index 4cad97cc..01aabb37 100755 --- a/lib/network/api_provider.dart +++ b/lib/network/api_provider.dart @@ -353,7 +353,7 @@ class ApiProvider extends BaseProvider { ); // 获取手机联网token - Future getLockNetToken(String lockId) => post( + Future getLockNetToken(int lockId) => post( getLockNetTokenURL.toUrl, jsonEncode({ 'lockId': lockId, diff --git a/lib/network/api_repository.dart b/lib/network/api_repository.dart index b5e09c33..86d3c604 100755 --- a/lib/network/api_repository.dart +++ b/lib/network/api_repository.dart @@ -325,7 +325,7 @@ class ApiRepository { } // 获取手机联网token - Future getLockNetToken({required String lockId}) async { + Future getLockNetToken({required int lockId}) async { final res = await apiProvider.getLockNetToken(lockId); return LockNetTokenEntity.fromJson(res.body); } diff --git a/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart b/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart index 44705624..028ffb73 100644 --- a/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart +++ b/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart @@ -62,9 +62,6 @@ class UdpTalkDataHandler extends ScpMessageBaseHandle int? spTotal, int? spIndex, int? messageId}) { - // 获取统计信息 - final stats = PacketLossStatistics().getStatistics(); - // _asyncLog('丢包统计: $stats'); // _asyncLog( // '分包数据:messageId:$messageId [$spIndex/$spTotal] PayloadLength:$PayloadLength'); if (messageType == MessageTypeConstant.RealTimeData) { diff --git a/lib/talk/starChart/handle/other/h264_frame_handler.dart b/lib/talk/starChart/handle/other/h264_frame_handler.dart index 903524da..face6b68 100644 --- a/lib/talk/starChart/handle/other/h264_frame_handler.dart +++ b/lib/talk/starChart/handle/other/h264_frame_handler.dart @@ -10,9 +10,6 @@ import '../../proto/talk_data_h264_frame.pb.dart'; class H264FrameHandler { final void Function(TalkDataModel frameData) onCompleteFrame; - // 只记录最近一个I帧的序号 - int _lastProcessedIFrameSeq = -1; - H264FrameHandler({required this.onCompleteFrame}); void handleFrame(TalkDataH264Frame frame, TalkData talkData) { diff --git a/lib/talk/starChart/handle/other/packet_loss_statistics.dart b/lib/talk/starChart/handle/other/packet_loss_statistics.dart index 34dd2282..8aabf6fd 100644 --- a/lib/talk/starChart/handle/other/packet_loss_statistics.dart +++ b/lib/talk/starChart/handle/other/packet_loss_statistics.dart @@ -10,6 +10,10 @@ class PacketLossStatistics { // key: messageId, value: {totalPackets, receivedPackets} final Map _packetsMap = HashMap(); + // 配置参数 + int _maxCapacity = 300; // 最大容量为300条记录 + int _timeoutMs = 30000; // 默认超时时间为30秒 + // 统计信息 int _totalMessages = 0; // 总消息数 int _lostMessages = 0; // 丢包的消息数 @@ -18,10 +22,19 @@ class PacketLossStatistics { // 记录分包数据 void recordPacket(int messageId, int currentIndex, int totalPackets) { + // 定期清理超时记录 + _cleanupExpiredPackets(); + + // 检查容量限制 + _checkCapacityLimit(); + if (!_packetsMap.containsKey(messageId)) { _packetsMap[messageId] = PacketInfo(totalPackets); _totalMessages++; _totalPackets += totalPackets; + } else { + // 更新时间戳 + _packetsMap[messageId]!.timestamp = DateTime.now().millisecondsSinceEpoch; } _packetsMap[messageId]!.receivedPackets.add(currentIndex); @@ -32,6 +45,51 @@ class PacketLossStatistics { } } + // 清理超时的记录 + void _cleanupExpiredPackets() { + final currentTime = DateTime.now().millisecondsSinceEpoch; + final expiredMessageIds = []; + + _packetsMap.forEach((messageId, info) { + // 如果记录超时，添加到待清理列表 + if (currentTime - info.timestamp > _timeoutMs) { + expiredMessageIds.add(messageId); + + // 统计丢包 + _lostMessages++; + _lostPackets += (info.totalPackets - info.receivedPackets.length); + } + }); + + // 移除超时记录 + for (var messageId in expiredMessageIds) { + _packetsMap.remove(messageId); + } + } + + // 检查并确保不超过最大容量 + void _checkCapacityLimit() { + if (_packetsMap.length <= _maxCapacity) { + return; + } + + // 如果超过容量限制，按时间戳排序并删除最旧的记录 + var entries = _packetsMap.entries.toList() + ..sort((a, b) => a.value.timestamp.compareTo(b.value.timestamp)); + + // 计算需要移除的数量（移除25%的旧记录，至少保证有一定空间） + int removeCount = (_packetsMap.length * 0.25).ceil(); + + // 移除并统计丢包 + for (int i = 0; i < removeCount && i < entries.length; i++) { + var entry = entries[i]; + _lostMessages++; + _lostPackets += + (entry.value.totalPackets - entry.value.receivedPackets.length); + _packetsMap.remove(entry.key); + } + } + // 检查丢包情况 void _checkPacketLoss(int messageId) { final info = _packetsMap[messageId]!; @@ -62,6 +120,28 @@ class PacketLossStatistics { return PacketLossInfo(messageLossRate, packetLossRate); } + // Getter和Setter，允许外部调整参数 + int get maxCapacity => _maxCapacity; + set maxCapacity(int value) { + if (value > 0) { + _maxCapacity = value; + // 设置新容量后立即检查 + _checkCapacityLimit(); + } + } + + int get timeoutMs => _timeoutMs; + set timeoutMs(int value) { + if (value > 0) { + _timeoutMs = value; + // 设置新超时后立即清理 + _cleanupExpiredPackets(); + } + } + + // 获取当前未完成记录数 + int get pendingRecordsCount => _packetsMap.length; + // 重置统计数据 void reset() { _packetsMap.clear(); @@ -76,8 +156,10 @@ class PacketLossStatistics { class PacketInfo { final int totalPackets; final Set receivedPackets = HashSet(); + int timestamp; // 添加时间戳字段，记录最后更新时间 - PacketInfo(this.totalPackets); + PacketInfo(this.totalPackets) + : timestamp = DateTime.now().millisecondsSinceEpoch; } // 丢包统计信息类 diff --git a/lib/talk/starChart/star_chart_manage.dart b/lib/talk/starChart/star_chart_manage.dart index a8f9ad5f..66de4aeb 100644 --- a/lib/talk/starChart/star_chart_manage.dart +++ b/lib/talk/starChart/star_chart_manage.dart @@ -1226,7 +1226,7 @@ class StartChartManage { await Storage.removerStarChartRegisterNodeInfo(); // 关闭udp服务 closeUdpSocket(); - PacketLossStatistics().reset(); + } /// 重置数据 diff --git a/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart b/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart new file mode 100644 index 00000000..cf76414e --- /dev/null +++ b/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart @@ -0,0 +1,798 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:ui' as ui; +import 'dart:math'; // Import the math package to use sqrt + +import 'package:flutter/foundation.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_pcm_sound/flutter_pcm_sound.dart'; +import 'package:flutter_voice_processor/flutter_voice_processor.dart'; +import 'package:gallery_saver/gallery_saver.dart'; +import 'package:get/get.dart'; +import 'package:image_gallery_saver/image_gallery_saver.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:permission_handler/permission_handler.dart'; +import 'package:star_lock/app_settings/app_settings.dart'; +import 'package:star_lock/login/login/entity/LoginEntity.dart'; +import 'package:star_lock/main/lockDetail/lockDetail/lockDetail_logic.dart'; +import 'package:star_lock/main/lockDetail/lockDetail/lockDetail_state.dart'; +import 'package:star_lock/main/lockDetail/lockDetail/lockNetToken_entity.dart'; +import 'package:star_lock/main/lockDetail/lockSet/lockSet/lockSetInfo_entity.dart'; +import 'package:star_lock/main/lockMian/entity/lockListInfo_entity.dart'; +import 'package:star_lock/network/api_repository.dart'; +import 'package:star_lock/talk/call/callTalk.dart'; +import 'package:star_lock/talk/call/g711.dart'; +import 'package:star_lock/talk/starChart/constant/talk_status.dart'; +import 'package:star_lock/talk/starChart/handle/other/packet_loss_statistics.dart'; +import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; +import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; +import 'package:star_lock/talk/starChart/proto/talk_data_h264_frame.pb.dart'; +import 'package:star_lock/talk/starChart/proto/talk_expect.pb.dart'; +import 'package:star_lock/talk/starChart/star_chart_manage.dart'; +import 'package:star_lock/talk/starChart/views/native/talk_view_native_decode_state.dart'; +import 'package:star_lock/talk/starChart/views/talkView/talk_view_state.dart'; +import 'package:star_lock/tools/G711Tool.dart'; +import 'package:star_lock/tools/bugly/bugly_tool.dart'; +import 'package:video_decode_plugin/video_decode_plugin.dart'; + +import '../../../../tools/baseGetXController.dart'; + +class TalkViewNativeDecodeLogic extends BaseGetXController { + final TalkViewNativeDecodeState state = TalkViewNativeDecodeState(); + + final LockDetailState lockDetailState = Get.put(LockDetailLogic()).state; + + int bufferSize = 25; // 初始化为默认大小 + + int audioBufferSize = 2; // 音频默认缓冲2帧 + + // 定义音频帧缓冲和发送函数 + final List _bufferedAudioFrames = []; + + // 添加监听状态和订阅引用 + bool _isListening = false; + StreamSubscription? _streamSubscription; + + Timer? _batchProcessTimer; + + // 添加一个集合来跟踪已成功解码的I帧序号 + final Set _decodedIFrames = {}; + + // 初始化视频解码器 + Future _initVideoDecoder() async { + try { + // 创建解码器配置 + final config = VideoDecoderConfig( + width: 864, + // 实际视频宽度 + height: 480, + frameRate: 25, + // 明确设置帧率 + // 增大缓冲区大小 + codecType: CodecType.h264, + // 编解码类型 + isDebug: true, + ); + + // 初始化解码器并获取textureId + final textureId = await VideoDecodePlugin.initDecoder(config); + + if (textureId != null) { + state.textureId.value = textureId; + AppLog.log('视频解码器初始化成功：textureId=$textureId'); + + // 设置帧回调 + VideoDecodePlugin.setFrameCallback(_onFrameAvailable); + + // 设置状态回调 + VideoDecodePlugin.setStateCallbackForTexture( + textureId, _onDecoderStateChanged); + + // 启动FPS监测 + startFpsMonitoring(); + } else { + AppLog.log('视频解码器初始化失败'); + } + } catch (e) { + AppLog.log('初始化视频解码器错误: $e'); + // 如果初始化失败，延迟后重试 + await Future.delayed(const Duration(seconds: 2)); + if (!Get.isRegistered()) { + return; // 如果控制器已经被销毁，不再重试 + } + _initVideoDecoder(); // 重试初始化 + } + } + + // 添加帧可用回调 + void _onFrameAvailable(int textureId) {} + + // 解码器状态变化回调 + void _onDecoderStateChanged( + int textureId, DecoderState decoderState, Map stats) { + String stateText; + switch (decoderState) { + case DecoderState.initializing: + state.isLoading.value = true; + stateText = "初始化中"; + break; + case DecoderState.ready: + stateText = "准备就绪"; + break; + case DecoderState.rendering: + stateText = "渲染中"; + state.isLoading.value = false; + break; + case DecoderState.error: + stateText = "出错"; + // 获取错误信息 + final errorMessage = stats['errorMessage'] as String?; + if (errorMessage != null) { + AppLog.log("解码器错误: $errorMessage"); + } + break; + case DecoderState.released: + stateText = "已释放"; + break; + default: + stateText = "未知状态"; + } + + // 更新统计信息 + if (stats.isNotEmpty) { + // 获取丢包率统计信息 + final PacketLossInfo packetLossInfo = + PacketLossStatistics().getStatistics(); + + // 更新FPS + // state.decoderFps.value = (stats['fps'] as num?)?.toDouble() ?? 0.0; + // 更新解码器详细信息 + state.renderedFrameCount.value = (stats['renderedFrames'] as int?) ?? 0; + state.totalFrames.value = (stats['totalFrames'] as int?) ?? 0; + state.droppedFrames.value = (stats['droppedFrames'] as int?) ?? 0; + state.hasSentIDR.value = (stats['hasSentIDR'] as bool?) ?? false; + state.hasSentSPS.value = (stats['hasSentSPS'] as bool?) ?? false; + state.hasSentPPS.value = (stats['hasSentPPS'] as bool?) ?? false; + state.keyFrameInterval.value = (stats['keyFrameInterval'] as int?) ?? 0; + state.decodingJitterMs.value = (stats['decodingJitterMs'] as int?) ?? 0; + + // 更新状态数据 + state.messageLossRate.value = packetLossInfo.messageLossRate; + state.packetLossRate.value = packetLossInfo.packetLossRate; + state.lastPacketStatsUpdateTime.value = + DateTime.now().millisecondsSinceEpoch; + } + } + + /// 初始化音频播放器 + void _initFlutterPcmSound() { + const int sampleRate = 8000; + FlutterPcmSound.setLogLevel(LogLevel.none); + FlutterPcmSound.setup(sampleRate: sampleRate, channelCount: 1); + // 设置 feed 阈值 + if (Platform.isAndroid) { + FlutterPcmSound.setFeedThreshold(1024); // Android 平台的特殊处理 + } else { + FlutterPcmSound.setFeedThreshold(2000); // 非 Android 平台的处理 + } + } + + /// 挂断 + void udpHangUpAction() async { + if (state.talkStatus.value == TalkStatus.answeredSuccessfully) { + // 如果是通话中就挂断 + StartChartManage().startTalkHangupMessageTimer(); + } else { + // 拒绝 + StartChartManage().startTalkRejectMessageTimer(); + } + if (state.textureId.value != null) { + VideoDecodePlugin.releaseDecoderForTexture(state.textureId.value!); + } + VideoDecodePlugin.releaseAllDecoders(); + Get.back(); + } + + // 发起接听命令 + void initiateAnswerCommand() { + StartChartManage().startTalkAcceptTimer(); + } + + // 监听音视频数据流 + void _startListenTalkData() { + // 防止重复监听 + if (_isListening) { + AppLog.log("已经存在数据流监听，避免重复监听"); + return; + } + + AppLog.log("==== 启动新的数据流监听 ===="); + _isListening = true; + + _streamSubscription = state.talkDataRepository.talkDataStream + .listen((TalkDataModel talkDataModel) async { + final talkData = talkDataModel.talkData; + final talkDataH264Frame = talkDataModel.talkDataH264Frame; + final contentType = talkData!.contentType; + + // 判断数据类型，进行分发处理 + switch (contentType) { + case TalkData_ContentTypeE.G711: + if (state.audioBuffer.length >= audioBufferSize) { + state.audioBuffer.removeAt(0); // 丢弃最旧的数据 + } + state.audioBuffer.add(talkData); // 添加新数据 + // 添加音频播放逻辑，与视频类似 + _playAudioFrames(); + break; + case TalkData_ContentTypeE.H264: + // 添加到视频帧缓冲区，而不是直接处理 + // _processH264Frame(talkData, talkDataH264Frame!); + // 直接处理H264视频帧 + _processH264Frame(talkData, talkDataH264Frame!); + + // 记录关键调试信息 + if (talkDataH264Frame!.frameType == TalkDataH264Frame_FrameTypeE.I) { + AppLog.log( + '帧序号${talkDataH264Frame.frameSeq};帧类型:${talkDataH264Frame.frameType.toString()};时间戳:${DateTime.now().millisecondsSinceEpoch}'); + } + break; + } + }); + } + + // 处理H264视频帧 + Future _processH264Frame( + TalkData talkData, TalkDataH264Frame frameInfo) async { + // 检查解码器是否已初始化 + if (state.textureId.value == null) { + // 可以记录日志或尝试初始化解码器 + AppLog.log('解码器尚未初始化，尝试重新初始化...'); + await _initVideoDecoder(); + + // 如果仍未初始化成功，则丢弃此帧 + if (state.textureId.value == null) { + return; + } + } + // 获取P帧对应的I帧序号 + final frameSeqI = frameInfo.frameSeqI; + + // P帧检查：如果依赖的I帧未解码成功，直接丢弃 + if (frameInfo.frameType == TalkDataH264Frame_FrameTypeE.P && + !_decodedIFrames.contains(frameSeqI)) { + AppLog.log('丢弃P帧: 依赖的I帧(${frameSeqI})尚未解码, P帧序号: ${frameInfo.frameSeq}'); + return; + } + + // 从talkData中提取H264帧数据 + final Uint8List frameData = Uint8List.fromList(talkData.content); + + // 确定帧类型 + final FrameType frameType = + frameInfo.frameType == TalkDataH264Frame_FrameTypeE.I + ? FrameType.iFrame + : FrameType.pFrame; + + // 将帧数据交给解码器处理 + try { + final bool result = + await VideoDecodePlugin.decodeFrame(frameData, frameType); + // 如果是I帧且成功解码，将其序号加入已解码I帧集合 + if (frameInfo.frameType == TalkDataH264Frame_FrameTypeE.I && result) { + _decodedIFrames.add(frameInfo.frameSeq); + // 限制集合大小，避免内存泄漏 + if (_decodedIFrames.length > 30) { + _decodedIFrames.remove(_decodedIFrames.first); + } + } + } catch (e) { + AppLog.log('解码帧错误: $e, 帧序号: ${frameInfo.frameSeq}'); + } + } + + // 新增：音频帧播放逻辑 + void _playAudioFrames() { + // 如果缓冲区为空或未达到目标大小，不进行播放 + // 音频缓冲区要求更小，以减少延迟 + if (state.audioBuffer.isEmpty || + state.audioBuffer.length < audioBufferSize) { + return; + } + + // 找出时间戳最小的音频帧 + TalkData? oldestFrame; + int oldestIndex = -1; + for (int i = 0; i < state.audioBuffer.length; i++) { + if (oldestFrame == null || + state.audioBuffer[i].durationMs < oldestFrame.durationMs) { + oldestFrame = state.audioBuffer[i]; + oldestIndex = i; + } + } + + // 确保找到了有效帧 + if (oldestFrame != null && oldestIndex != -1) { + if (state.isOpenVoice.value) { + // 播放音频 + _playAudioData(oldestFrame); + } + state.audioBuffer.removeAt(oldestIndex); + } + } + + /// 监听对讲状态 + void _startListenTalkStatus() { + state.startChartTalkStatus.statusStream.listen((talkStatus) { + state.talkStatus.value = talkStatus; + switch (talkStatus) { + case TalkStatus.rejected: + case TalkStatus.hangingUpDuring: + case TalkStatus.notTalkData: + case TalkStatus.notTalkPing: + case TalkStatus.end: + _handleInvalidTalkStatus(); + break; + case TalkStatus.answeredSuccessfully: + state.oneMinuteTimeTimer?.cancel(); // 取消旧定时器 + state.oneMinuteTimeTimer ??= + Timer.periodic(const Duration(seconds: 1), (Timer t) { + if (state.isLoading.isFalse) { + state.oneMinuteTime.value++; + } + }); + break; + default: + // 其他状态的处理 + break; + } + }); + } + + /// 播放音频数据 + void _playAudioData(TalkData talkData) async { + if (state.isOpenVoice.value) { + final list = + G711().decodeAndDenoise(talkData.content, true, 8000, 300, 150); + // // 将 PCM 数据转换为 PcmArrayInt16 + final PcmArrayInt16 fromList = PcmArrayInt16.fromList(list); + FlutterPcmSound.feed(fromList); + if (!state.isPlaying.value) { + FlutterPcmSound.play(); + state.isPlaying.value = true; + } + } + } + + /// 停止播放音频 + void _stopPlayG711Data() async { + await FlutterPcmSound.pause(); + await FlutterPcmSound.stop(); + await FlutterPcmSound.clear(); + } + + /// 获取权限状态 + Future getPermissionStatus() async { + final Permission permission = Permission.microphone; + //granted 通过，denied 被拒绝，permanentlyDenied 拒绝且不在提示 + final PermissionStatus status = await permission.status; + if (status.isGranted) { + return true; + } else if (status.isDenied) { + requestPermission(permission); + } else if (status.isPermanentlyDenied) { + openAppSettings(); + } else if (status.isRestricted) { + requestPermission(permission); + } else {} + return false; + } + + ///申请权限 + void requestPermission(Permission permission) async { + final PermissionStatus status = await permission.request(); + if (status.isPermanentlyDenied) { + openAppSettings(); + } + } + + Future requestPermissions() async { + // 申请存储权限 + var storageStatus = await Permission.storage.request(); + // 申请录音权限 + var microphoneStatus = await Permission.microphone.request(); + + if (storageStatus.isGranted && microphoneStatus.isGranted) { + print("Permissions granted"); + } else { + print("Permissions denied"); + // 如果权限被拒绝，可以提示用户或跳转到设置页面 + if (await Permission.storage.isPermanentlyDenied) { + openAppSettings(); // 跳转到应用设置页面 + } + } + } + + Future startRecording() async {} + + Future stopRecording() async {} + + @override + void onReady() { + super.onReady(); + } + + @override + void onInit() { + super.onInit(); + + // 启动监听音视频数据流 + _startListenTalkData(); + // 启动监听对讲状态 + _startListenTalkStatus(); + // 在没有监听成功之前赋值一遍状态 + // *** 由于页面会在状态变化之后才会初始化，导致识别不到最新的状态，在这里手动赋值 *** + state.talkStatus.value = state.startChartTalkStatus.status; + + // 初始化音频播放器 + _initFlutterPcmSound(); + + // 启动播放定时器 + // _startPlayback(); + + // 初始化录音控制器 + _initAudioRecorder(); + + requestPermissions(); + + // 初始化视频解码器 + _initVideoDecoder(); + } + + @override + void onClose() { + _stopPlayG711Data(); // 停止播放音频 + + state.audioBuffer.clear(); // 清空音频缓冲区 + + state.oneMinuteTimeTimer?.cancel(); + state.oneMinuteTimeTimer = null; + + // 停止播放音频 + stopProcessingAudio(); + + state.oneMinuteTimeTimer?.cancel(); // 取消旧定时器 + state.oneMinuteTimeTimer = null; // 取消旧定时器 + state.oneMinuteTime.value = 0; + + // 释放视频解码器资源 + if (state.textureId.value != null) { + VideoDecodePlugin.releaseDecoder(); + state.textureId.value = null; + } + + // 取消数据流监听 + _streamSubscription?.cancel(); + _isListening = false; + + // 停止FPS监测 + stopFpsMonitoring(); + // 重置期望数据 + StartChartManage().reSetDefaultTalkExpect(); + VideoDecodePlugin.releaseAllDecoders(); + + // 取消批处理定时器 + _batchProcessTimer?.cancel(); + _batchProcessTimer = null; + + // 清空已解码I帧集合 + _decodedIFrames.clear(); + + super.onClose(); + } + + /// 处理无效通话状态 + void _handleInvalidTalkStatus() { + // 停止播放音频 + _stopPlayG711Data(); + stopProcessingAudio(); + } + + /// 更新发送预期数据 + void updateTalkExpect() { + TalkExpectReq talkExpectReq = TalkExpectReq(); + state.isOpenVoice.value = !state.isOpenVoice.value; + if (!state.isOpenVoice.value) { + talkExpectReq = TalkExpectReq( + videoType: [VideoTypeE.IMAGE], + audioType: [], + ); + showToast('已静音'.tr); + } else { + talkExpectReq = TalkExpectReq( + videoType: [VideoTypeE.IMAGE], + audioType: [AudioTypeE.G711], + ); + } + + /// 修改发送预期数据 + StartChartManage().changeTalkExpectDataTypeAndReStartTalkExpectMessageTimer( + talkExpect: talkExpectReq); + } + + /// 截图并保存到相册 + Future captureAndSavePng() async { + try { + if (state.globalKey.currentContext == null) { + AppLog.log('截图失败: 未找到当前上下文'); + return; + } + final RenderRepaintBoundary boundary = state.globalKey.currentContext! + .findRenderObject()! as RenderRepaintBoundary; + final ui.Image image = await boundary.toImage(); + final ByteData? byteData = + await image.toByteData(format: ui.ImageByteFormat.png); + + if (byteData == null) { + AppLog.log('截图失败: 图像数据为空'); + return; + } + final Uint8List pngBytes = byteData.buffer.asUint8List(); + + // 获取应用程序的文档目录 + final Directory directory = await getApplicationDocumentsDirectory(); + final String imagePath = '${directory.path}/screenshot.png'; + + // 将截图保存为文件 + final File imgFile = File(imagePath); + await imgFile.writeAsBytes(pngBytes); + + // 将截图保存到相册 + await ImageGallerySaver.saveFile(imagePath); + + AppLog.log('截图保存路径: $imagePath'); + showToast('截图已保存到相册'.tr); + } catch (e) { + AppLog.log('截图失败: $e'); + } + } + + // 远程开锁 + Future remoteOpenLock() async { + final lockPeerId = StartChartManage().lockPeerId; + final lockListPeerId = StartChartManage().lockListPeerId; + int lockId = lockDetailState.keyInfos.value.lockId ?? 0; + + // 如果锁列表获取到peerId，代表有多个锁，使用锁列表的peerId + // 从列表中遍历出对应的peerId + lockListPeerId.forEach((element) { + if (element.network?.peerId == lockPeerId) { + lockId = element.lockId ?? 0; + } + }); + + final LockSetInfoEntity lockSetInfoEntity = + await ApiRepository.to.getLockSettingInfoData( + lockId: lockId.toString(), + ); + if (lockSetInfoEntity.errorCode!.codeIsSuccessful) { + if (lockSetInfoEntity.data?.lockFeature?.remoteUnlock == 1 && + lockSetInfoEntity.data?.lockSettingInfo?.remoteUnlock == 1) { + final LoginEntity entity = await ApiRepository.to + .remoteOpenLock(lockId: lockId.toString(), timeOut: 60); + if (entity.errorCode!.codeIsSuccessful) { + showToast('已开锁'.tr); + StartChartManage().lockListPeerId = []; + } + } else { + showToast('该锁的远程开锁功能未启用'.tr); + } + } + } + + /// 初始化音频录制器 + void _initAudioRecorder() { + state.voiceProcessor = VoiceProcessor.instance; + } + + //开始录音 + Future startProcessingAudio() async { + try { + if (await state.voiceProcessor?.hasRecordAudioPermission() ?? false) { + await state.voiceProcessor?.start(state.frameLength, state.sampleRate); + final bool? isRecording = await state.voiceProcessor?.isRecording(); + state.isRecordingAudio.value = isRecording!; + state.startRecordingAudioTime.value = DateTime.now(); + + // 增加录音帧监听器和错误监听器 + state.voiceProcessor + ?.addFrameListeners([_onFrame]); + state.voiceProcessor?.addErrorListener(_onError); + } else { + // state.errorMessage.value = 'Recording permission not granted'; + } + } on PlatformException catch (ex) { + // state.errorMessage.value = 'Failed to start recorder: $ex'; + } + state.isOpenVoice.value = false; + } + + /// 停止录音 + Future stopProcessingAudio() async { + try { + await state.voiceProcessor?.stop(); + state.voiceProcessor?.removeFrameListener(_onFrame); + state.udpSendDataFrameNumber = 0; + // 记录结束时间 + state.endRecordingAudioTime.value = DateTime.now(); + + // 计算录音的持续时间 + final Duration duration = state.endRecordingAudioTime.value + .difference(state.startRecordingAudioTime.value); + + state.recordingAudioTime.value = duration.inSeconds; + } on PlatformException catch (ex) { + // state.errorMessage.value = 'Failed to stop recorder: $ex'; + } finally { + final bool? isRecording = await state.voiceProcessor?.isRecording(); + state.isRecordingAudio.value = isRecording!; + state.isOpenVoice.value = true; + } + } + +// 音频帧处理 + Future _onFrame(List frame) async { + // 添加最大缓冲限制 + if (_bufferedAudioFrames.length > state.frameLength * 3) { + _bufferedAudioFrames.clear(); // 清空过多积累的数据 + return; + } + + // 首先应用固定增益提升基础音量 + List amplifiedFrame = _applyGain(frame, 1.6); + // 编码为G711数据 + List encodedData = G711Tool.encode(amplifiedFrame, 0); // 0表示A-law + _bufferedAudioFrames.addAll(encodedData); + // 使用相对时间戳 + final int ms = DateTime.now().millisecondsSinceEpoch % 1000000; // 使用循环时间戳 + int getFrameLength = state.frameLength; + if (Platform.isIOS) { + getFrameLength = state.frameLength * 2; + } + + // 添加发送间隔控制 + if (_bufferedAudioFrames.length >= state.frameLength) { + try { + await StartChartManage().sendTalkDataMessage( + talkData: TalkData( + content: _bufferedAudioFrames, + contentType: TalkData_ContentTypeE.G711, + durationMs: ms, + ), + ); + } finally { + _bufferedAudioFrames.clear(); // 确保清理缓冲区 + } + } else { + _bufferedAudioFrames.addAll(encodedData); + } + } + +// 错误监听 + void _onError(VoiceProcessorException error) { + AppLog.log(error.message!); + } + +// 添加音频增益处理方法 + List _applyGain(List pcmData, double gainFactor) { + List result = List.filled(pcmData.length, 0); + + for (int i = 0; i < pcmData.length; i++) { + // PCM数据通常是有符号的16位整数 + int sample = pcmData[i]; + + // 应用增益 + double amplified = sample * gainFactor; + + // 限制在有效范围内，防止溢出 + if (amplified > 32767) { + amplified = 32767; + } else if (amplified < -32768) { + amplified = -32768; + } + + result[i] = amplified.toInt(); + } + + return result; + } + + // 启动网络状态监测 + void startFpsMonitoring() { + // 确保只有一个计时器在运行 + stopFpsMonitoring(); + + // 初始化时间记录 + state.lastFpsUpdateTime.value = DateTime.now().millisecondsSinceEpoch; + + // 创建一个计时器，每秒更新一次丢包率和性能数据 + state.fpsTimer = Timer.periodic(const Duration(seconds: 1), (timer) { + // 更新丢包率数据 + updatePacketLossStats(); + + // 分析性能数据 + _analyzePerformance(); + }); + } + + // 停止网络状态监测 + void stopFpsMonitoring() { + state.fpsTimer?.cancel(); + state.fpsTimer = null; + } + + // 日志记录方法 + void logMessage(String message) { + AppLog.log(message); + } + + // 更新丢包率统计数据 + void updatePacketLossStats() async { + try {} catch (e) { + logMessage('获取丢包率数据失败: $e'); + } + } + + // 添加性能分析方法 + void _analyzePerformance() { + final int now = DateTime.now().millisecondsSinceEpoch; + + // 如果是首次调用，初始化数据 + if (state.lastPerformanceCheck == 0) { + state.lastPerformanceCheck = now; + state.lastFrameCount = state.renderedFrameCount.value; + return; + } + + // 每秒分析一次性能 + if (now - state.lastPerformanceCheck >= 1000) { + // 计算过去一秒的实际帧率 + final int frameRendered = + state.renderedFrameCount.value - state.lastFrameCount; + final double actualFPS = + frameRendered * 1000 / (now - state.lastPerformanceCheck); + + // 计算丢帧率 + final double dropRate = state.droppedFrames.value / + (state.totalFrames.value > 0 ? state.totalFrames.value : 1) * + 100; + + // 计算当前解码器积压帧数 + final int pendingFrames = + state.totalFrames.value - state.renderedFrameCount.value; + + // 计算跟踪Map中的帧数（正在处理中的帧） + final int processingFrames = state.frameTracker.length; + + // 分析渲染瓶颈 + String performanceStatus = "正常"; + if (actualFPS < 15 && dropRate > 10) { + performanceStatus = "严重渲染瓶颈"; + } else if (actualFPS < 20 && dropRate > 5) { + performanceStatus = "轻微渲染瓶颈"; + } + + // 输出综合性能分析 + AppLog.log("性能分析: 实际帧率=${actualFPS.toStringAsFixed(1)}fps, " + + "丢帧率=${dropRate.toStringAsFixed(1)}%, " + + "待处理帧数=$pendingFrames, " + + "处理中帧数=$processingFrames, " + + "状态=$performanceStatus"); + + // 重置统计数据 + state.lastPerformanceCheck = now; + state.lastFrameCount = state.renderedFrameCount.value; + } + } +} diff --git a/lib/talk/starChart/views/native/talk_view_native_decode_page.dart b/lib/talk/starChart/views/native/talk_view_native_decode_page.dart new file mode 100644 index 00000000..3172a6a6 --- /dev/null +++ b/lib/talk/starChart/views/native/talk_view_native_decode_page.dart @@ -0,0 +1,557 @@ +import 'dart:async'; +import 'dart:math'; + +import 'package:flutter/material.dart'; +import 'package:flutter/scheduler.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_screenutil/flutter_screenutil.dart'; +import 'package:get/get.dart'; +import 'package:http/http.dart' as http; +import 'package:provider/provider.dart'; +import 'package:star_lock/flavors.dart'; +import 'package:star_lock/talk/call/callTalk.dart'; +import 'package:star_lock/talk/starChart/constant/talk_status.dart'; +import 'package:star_lock/talk/starChart/handle/impl/debug_Info_model.dart'; +import 'package:star_lock/talk/starChart/handle/impl/udp_talk_data_handler.dart'; +import 'package:star_lock/talk/starChart/views/native/talk_view_native_decode_logic.dart'; +import 'package:star_lock/talk/starChart/views/native/talk_view_native_decode_state.dart'; +import 'package:star_lock/talk/starChart/views/talkView/talk_view_logic.dart'; +import 'package:star_lock/talk/starChart/views/talkView/talk_view_state.dart'; +import 'package:video_decode_plugin/video_decode_plugin.dart'; + +import '../../../../app_settings/app_colors.dart'; +import '../../../../tools/showTFView.dart'; + +class TalkViewNativeDecodePage extends StatefulWidget { + const TalkViewNativeDecodePage({Key? key}) : super(key: key); + + @override + State createState() => + _TalkViewNativeDecodePageState(); +} + +class _TalkViewNativeDecodePageState extends State + with TickerProviderStateMixin { + final TalkViewNativeDecodeLogic logic = Get.put(TalkViewNativeDecodeLogic()); + final TalkViewNativeDecodeState state = + Get.find().state; + + @override + void initState() { + super.initState(); + + state.animationController = AnimationController( + vsync: this, // 确保使用的TickerProvider是当前Widget + duration: const Duration(seconds: 1), + ); + + state.animationController.repeat(); + //动画开始、结束、向前移动或向后移动时会调用StatusListener + state.animationController.addStatusListener((AnimationStatus status) { + if (status == AnimationStatus.completed) { + state.animationController.reset(); + state.animationController.forward(); + } else if (status == AnimationStatus.dismissed) { + state.animationController.reset(); + state.animationController.forward(); + } + }); + } + + @override + Widget build(BuildContext context) { + return WillPopScope( + onWillPop: () async { + // 返回 false 表示禁止退出 + return false; + }, + child: SizedBox( + width: 1.sw, + height: 1.sh, + child: Stack( + alignment: Alignment.center, + children: [ + Obx( + () { + final double screenWidth = MediaQuery.of(context).size.width; + final double screenHeight = MediaQuery.of(context).size.height; + + final double logicalWidth = MediaQuery.of(context).size.width; + final double logicalHeight = MediaQuery.of(context).size.height; + final double devicePixelRatio = + MediaQuery.of(context).devicePixelRatio; + + // 计算物理像素值 + final double physicalWidth = logicalWidth * devicePixelRatio; + final double physicalHeight = logicalHeight * devicePixelRatio; + + // 旋转后的图片尺寸 + const int rotatedImageWidth = 480; // 原始高度 + const int rotatedImageHeight = 864; // 原始宽度 + + // 计算缩放比例 + final double scaleWidth = physicalWidth / rotatedImageWidth; + final double scaleHeight = physicalHeight / rotatedImageHeight; + max(scaleWidth, scaleHeight); // 选择较大的缩放比例 + + return state.isLoading.isTrue + ? Image.asset( + 'images/main/monitorBg.png', + width: screenWidth, + height: screenHeight, + fit: BoxFit.cover, + ) + : PopScope( + canPop: false, + child: RepaintBoundary( + key: state.globalKey, + child: SizedBox.expand( + child: RotatedBox( + // 解码器不支持硬件旋转，使用RotatedBox + quarterTurns: -1, + child: Texture( + textureId: state.textureId.value!, + filterQuality: FilterQuality.medium, + ), + ), + ), + ), + ); + }, + ), + Obx(() => state.isLoading.isTrue + ? Positioned( + bottom: 310.h, + child: Text( + '正在创建安全连接...'.tr, + style: TextStyle(color: Colors.black, fontSize: 26.sp), + )) + : Container()), + Obx(() => state.textureId.value != null && state.showFps.value + ? Positioned( + top: ScreenUtil().statusBarHeight + 10.h, + right: 20.w, + child: Container( + padding: + EdgeInsets.symmetric(horizontal: 10.w, vertical: 5.h), + decoration: BoxDecoration( + color: Colors.black.withOpacity(0.5), + borderRadius: BorderRadius.circular(5.h), + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.end, + children: [ + // Text( + // 'FPS: ${state.decoderFps.value.toStringAsFixed(1)}', + // style: TextStyle( + // color: _getPacketLossColor( + // state.packetLossRate.value), + // fontSize: 20.sp, + // ), + // ), + Text( + '丢包率: ${state.packetLossRate.value.toStringAsFixed(1)}%', + style: TextStyle( + color: _getPacketLossColor( + state.packetLossRate.value), + fontSize: 20.sp, + ), + ), + Text( + '消息丢失: ${state.messageLossRate.value.toStringAsFixed(1)}%', + style: TextStyle( + color: _getPacketLossColor( + state.messageLossRate.value), + fontSize: 20.sp, + ), + ), + Divider( + color: Colors.white30, + height: 10.h, + thickness: 1), + Text( + '已渲染帧: ${state.renderedFrameCount.value}', + style: + TextStyle(color: Colors.white, fontSize: 18.sp), + ), + Text( + '总帧数: ${state.totalFrames.value}', + style: + TextStyle(color: Colors.white, fontSize: 18.sp), + ), + Text( + '丢弃帧: ${state.droppedFrames.value}', + style: + TextStyle(color: Colors.white, fontSize: 18.sp), + ), + Text( + 'IDR帧: ${state.hasSentIDR.value ? "已发送" : "未发送"}', + style: TextStyle( + color: state.hasSentIDR.value + ? Colors.green + : Colors.red, + fontSize: 18.sp), + ), + Text( + 'SPS: ${state.hasSentSPS.value ? "已发送" : "未发送"}', + style: TextStyle( + color: state.hasSentSPS.value + ? Colors.green + : Colors.red, + fontSize: 18.sp), + ), + Text( + 'PPS: ${state.hasSentPPS.value ? "已发送" : "未发送"}', + style: TextStyle( + color: state.hasSentPPS.value + ? Colors.green + : Colors.red, + fontSize: 18.sp), + ), + Text( + 'keyFrameInterval: ${state.keyFrameInterval.value}', + style: + TextStyle(color: Colors.green, fontSize: 18.sp), + ), + Text( + 'decodingJitterMs: ${state.decodingJitterMs.value}', + style: + TextStyle(color: Colors.green, fontSize: 18.sp), + ), + ], + ), + ), + ) + : Container()), + Obx(() => state.isLoading.isFalse && state.oneMinuteTime.value > 0 + ? Positioned( + top: ScreenUtil().statusBarHeight + 75.h, + width: 1.sw, + child: Obx( + () { + final String sec = (state.oneMinuteTime.value % 60) + .toString() + .padLeft(2, '0'); + final String min = (state.oneMinuteTime.value ~/ 60) + .toString() + .padLeft(2, '0'); + return Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Text( + '$min:$sec', + style: TextStyle( + fontSize: 26.sp, color: Colors.white), + ), + ], + ); + }, + ), + ) + : Container()), + Positioned( + bottom: 10.w, + child: Container( + width: 1.sw - 30.w * 2, + // height: 300.h, + margin: EdgeInsets.all(30.w), + decoration: BoxDecoration( + color: Colors.black.withOpacity(0.2), + borderRadius: BorderRadius.circular(20.h)), + child: Column( + children: [ + SizedBox(height: 20.h), + bottomTopBtnWidget(), + SizedBox(height: 20.h), + bottomBottomBtnWidget(), + SizedBox(height: 20.h), + ], + ), + ), + ), + Obx(() => state.isLoading.isTrue + ? buildRotationTransition() + : Container()), + Obx(() => state.isLongPressing.value + ? Positioned( + top: 80.h, + left: 0, + right: 0, + child: Center( + child: Container( + padding: EdgeInsets.all(10.w), + decoration: BoxDecoration( + color: Colors.black.withOpacity(0.7), + borderRadius: BorderRadius.circular(10.w), + ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(Icons.mic, color: Colors.white, size: 24.w), + SizedBox(width: 10.w), + Text( + '正在说话...'.tr, + style: TextStyle( + fontSize: 20.sp, color: Colors.white), + ), + ], + ), + ), + ), + ) + : Container()), + ], + ), + ), + ); + } + + Widget bottomTopBtnWidget() { + return Row(mainAxisAlignment: MainAxisAlignment.center, children: [ + // 打开关闭声音 + GestureDetector( + onTap: () { + if (state.talkStatus.value == TalkStatus.answeredSuccessfully) { + // 打开关闭声音 + logic.updateTalkExpect(); + } + }, + child: Container( + width: 50.w, + height: 50.w, + padding: EdgeInsets.all(5.w), + child: Obx(() => Image( + width: 40.w, + height: 40.w, + image: state.isOpenVoice.value + ? const AssetImage( + 'images/main/icon_lockDetail_monitoringOpenVoice.png') + : const AssetImage( + 'images/main/icon_lockDetail_monitoringCloseVoice.png'))), + ), + ), + SizedBox(width: 50.w), + // 截图 + GestureDetector( + onTap: () async { + if (state.talkStatus.value == TalkStatus.answeredSuccessfully) { + await logic.captureAndSavePng(); + } + }, + child: Container( + width: 50.w, + height: 50.w, + padding: EdgeInsets.all(5.w), + child: Image( + width: 40.w, + height: 40.w, + image: const AssetImage( + 'images/main/icon_lockDetail_monitoringScreenshot.png')), + ), + ), + SizedBox(width: 50.w), + // 录制 + GestureDetector( + onTap: () async { + logic.showToast('功能暂未开放'.tr); + // if ( + // state.talkStatus.value == TalkStatus.answeredSuccessfully) { + // if (state.isRecordingScreen.value) { + // await logic.stopRecording(); + // } else { + // await logic.startRecording(); + // } + // } + }, + child: Container( + width: 50.w, + height: 50.w, + padding: EdgeInsets.all(5.w), + child: Image( + width: 40.w, + height: 40.w, + fit: BoxFit.fill, + image: const AssetImage( + 'images/main/icon_lockDetail_monitoringScreenRecording.png'), + ), + ), + ), + ]); + } + + Widget bottomBottomBtnWidget() { + return Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + // 接听 + Obx( + () => bottomBtnItemWidget( + getAnswerBtnImg(), + getAnswerBtnName(), + Colors.white, + longPress: () async { + if (state.talkStatus.value == TalkStatus.answeredSuccessfully) { + // 启动录音 + logic.startProcessingAudio(); + state.isLongPressing.value = true; + } + }, + longPressUp: () async { + // 停止录音 + logic.stopProcessingAudio(); + state.isLongPressing.value = false; + }, + onClick: () async { + if (state.talkStatus.value == + TalkStatus.passiveCallWaitingAnswer) { + // 接听 + logic.initiateAnswerCommand(); + } + }, + ), + ), + bottomBtnItemWidget( + 'images/main/icon_lockDetail_hangUp.png', '挂断'.tr, Colors.red, + onClick: () { + // 挂断 + logic.udpHangUpAction(); + }), + bottomBtnItemWidget( + 'images/main/icon_lockDetail_monitoringUnlock.png', + '开锁'.tr, + AppColors.mainColor, + onClick: () { + // if (state.talkStatus.value == TalkStatus.answeredSuccessfully && + // state.listData.value.length > 0) { + // logic.udpOpenDoorAction(); + // } + // if (UDPManage().remoteUnlock == 1) { + // logic.udpOpenDoorAction(); + // showDeletPasswordAlertDialog(context); + // } else { + // logic.showToast('请在锁设置中开启远程开锁'.tr); + // } + logic.remoteOpenLock(); + }, + ) + ]); + } + + String getAnswerBtnImg() { + switch (state.talkStatus.value) { + case TalkStatus.passiveCallWaitingAnswer: + return 'images/main/icon_lockDetail_monitoringAnswerCalls.png'; + case TalkStatus.answeredSuccessfully: + case TalkStatus.proactivelyCallWaitingAnswer: + return 'images/main/icon_lockDetail_monitoringUnTalkback.png'; + default: + return 'images/main/icon_lockDetail_monitoringAnswerCalls.png'; + } + } + + String getAnswerBtnName() { + switch (state.talkStatus.value) { + case TalkStatus.passiveCallWaitingAnswer: + return '接听'.tr; + case TalkStatus.proactivelyCallWaitingAnswer: + case TalkStatus.answeredSuccessfully: + return '长按说话'.tr; + default: + return '接听'.tr; + } + } + + Widget bottomBtnItemWidget( + String iconUrl, + String name, + Color backgroundColor, { + required Function() onClick, + Function()? longPress, + Function()? longPressUp, + }) { + double wh = 80.w; + return GestureDetector( + onTap: onClick, + onLongPress: longPress, + onLongPressUp: longPressUp, + child: SizedBox( + height: 160.w, + width: 140.w, + child: Column( + crossAxisAlignment: CrossAxisAlignment.center, + children: [ + Container( + width: wh, + height: wh, + constraints: BoxConstraints( + minWidth: wh, + ), + decoration: BoxDecoration( + color: backgroundColor, + borderRadius: BorderRadius.circular((wh + 10.w * 2) / 2), + ), + padding: EdgeInsets.all(20.w), + child: Image.asset(iconUrl, fit: BoxFit.fitWidth), + ), + SizedBox(height: 20.w), + Text( + name, + style: TextStyle(fontSize: 20.sp, color: Colors.white), + textAlign: TextAlign.center, // 当文本超出指定行数时，使用省略号表示 + maxLines: 2, // 设置最大行数为1 + ) + ], + ), + ), + ); + } + + // 根据丢包率返回对应的颜色 + Color _getPacketLossColor(double lossRate) { + if (lossRate < 1.0) { + return Colors.green; // 丢包率低于1%显示绿色 + } else if (lossRate < 5.0) { + return Colors.yellow; // 丢包率1%-5%显示黄色 + } else if (lossRate < 10.0) { + return Colors.orange; // 丢包率5%-10%显示橙色 + } else { + return Colors.red; // 丢包率高于10%显示红色 + } + } + + //旋转动画 + Widget buildRotationTransition() { + return Positioned( + left: ScreenUtil().screenWidth / 2 - 220.w / 2, + top: ScreenUtil().screenHeight / 2 - 220.w / 2 - 150.h, + child: GestureDetector( + child: RotationTransition( + //设置动画的旋转中心 + alignment: Alignment.center, + //动画控制器 + turns: state.animationController, + //将要执行动画的子view + child: AnimatedOpacity( + opacity: 0.5, + duration: const Duration(seconds: 2), + child: Image.asset( + 'images/main/realTime_connecting.png', + width: 220.w, + height: 220.w, + ), + ), + ), + onTap: () { + state.animationController.forward(); + }, + ), + ); + } + + @override + void dispose() { + state.animationController.dispose(); + CallTalk().finishAVData(); + super.dispose(); + } +} diff --git a/lib/talk/starChart/views/native/talk_view_native_decode_state.dart b/lib/talk/starChart/views/native/talk_view_native_decode_state.dart new file mode 100644 index 00000000..d98d5d34 --- /dev/null +++ b/lib/talk/starChart/views/native/talk_view_native_decode_state.dart @@ -0,0 +1,109 @@ +import 'dart:async'; +import 'dart:typed_data'; +import 'dart:ui' as ui; +import 'dart:io'; + +import 'package:flutter/material.dart'; +import 'package:flutter_voice_processor/flutter_voice_processor.dart'; +import 'package:get/get.dart'; +import 'package:get/get_rx/get_rx.dart'; +import 'package:get/get_rx/src/rx_types/rx_types.dart'; +import 'package:get/state_manager.dart'; +import 'package:network_info_plus/network_info_plus.dart'; +import 'package:star_lock/talk/starChart/constant/talk_status.dart'; +import 'package:star_lock/talk/starChart/handle/other/packet_loss_statistics.dart'; +import 'package:star_lock/talk/starChart/handle/other/talk_data_repository.dart'; +import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; +import 'package:star_lock/talk/starChart/status/star_chart_talk_status.dart'; +import 'package:video_decode_plugin/video_decode_plugin.dart'; + +import '../../../../tools/storage.dart'; + +enum NetworkStatus { + normal, // 0 + lagging, // 1 + delayed, // 2 + packetLoss // 3 +} + +class TalkViewNativeDecodeState { + // 视频源最大帧率限制 + static const int maxSourceFps = 25; // 视频源最高支持25fps + + int udpSendDataFrameNumber = 0; // 帧序号 + // var isSenderAudioData = false.obs;// 是否要发送音频数据 + + Future userMobileIP = NetworkInfo().getWifiIP(); + Future userUid = Storage.getUid(); + + RxInt udpStatus = + 0.obs; //0：初始状态 1：等待监视 2： 3：监视中 4：呼叫成功 5：主角通话中 6：被叫通话 8：被叫通话中 9：长按说话 + TextEditingController passwordTF = TextEditingController(); + + RxList listAudioData = [].obs; //得到的音频流字节数据 + GlobalKey globalKey = GlobalKey(); + + Timer? oneMinuteTimeTimer; // 定时器超过60秒关闭当前界面 + RxInt oneMinuteTime = 0.obs; // 定时器秒数 + + // 定时器如果发送了接听的命令 而没收到回复就每秒重复发送10次 + late Timer answerTimer; + late Timer hangUpTimer; + late Timer openDoorTimer; + Timer? fpsTimer; + late AnimationController animationController; + + RxInt elapsedSeconds = 0.obs; + + // 星图对讲相关状态 + List audioBuffer = [].obs; + + RxBool isLoading = true.obs; // 是否在加载 + RxBool isPlaying = false.obs; // 是否开始播放 + Rx talkStatus = TalkStatus.none.obs; //星图对讲状态 + // 获取 startChartTalkStatus 的唯一实例 + final StartChartTalkStatus startChartTalkStatus = + StartChartTalkStatus.instance; + + // 通话数据流的单例流数据处理类 + final TalkDataRepository talkDataRepository = TalkDataRepository.instance; + + RxBool isOpenVoice = true.obs; // 是否打开声音 + RxBool isRecordingScreen = false.obs; // 是否录屏中 + RxBool isRecordingAudio = false.obs; // 是否录音中 + Rx startRecordingAudioTime = DateTime.now().obs; // 开始录音时间 + Rx endRecordingAudioTime = DateTime.now().obs; // 结束录音时间 + RxInt recordingAudioTime = 0.obs; // 录音时间持续时间 + late VoiceProcessor? voiceProcessor; // 音频处理器、录音 + final int frameLength = 320; //录音视频帧长度为640 + final int sampleRate = 8000; //录音频采样率为8000 + RxBool isLongPressing = false.obs; // 旋转角度（以弧度为单位） + // 视频解码器纹理ID + Rx textureId = Rx(null); + // FPS监测相关变量 + + RxInt lastFpsUpdateTime = 0.obs; // 上次FPS更新时间 + RxBool showFps = true.obs; // 是否显示FPS + // 丢包率统计相关变量 + RxDouble decoderFps = 0.0.obs; // 消息丢失率 + RxDouble messageLossRate = 0.0.obs; // 消息丢失率 + RxDouble packetLossRate = 0.0.obs; // 分包丢失率 + RxInt lastPacketStatsUpdateTime = 0.obs; // 上次更新丢包统计的时间 + + // 解码器详细统计信息 + RxInt renderedFrameCount = 0.obs; // 已渲染帧数 + RxInt totalFrames = 0.obs; // 总帧数 + RxInt droppedFrames = 0.obs; // 丢弃帧数 + RxBool hasSentIDR = false.obs; // 是否已发送IDR帧 + RxBool hasSentSPS = false.obs; // 是否已发送SPS + RxBool hasSentPPS = false.obs; // 是否已发送PPS + RxInt keyFrameInterval = 0.obs; // 关键帧间隔时间（ms） + RxInt decodingJitterMs = 0.obs; // 解码抖动时间（ms） + + // 性能分析变量 + int lastPerformanceCheck = 0; + int lastFrameCount = 0; + + // 帧跟踪Map，记录每个提交的帧，key为textureId_frameSeq + Map> frameTracker = {}; +} diff --git a/lib/talk/starChart/views/talkView/talk_view_logic.dart b/lib/talk/starChart/views/talkView/talk_view_logic.dart index a4a850ba..eaf051c1 100644 --- a/lib/talk/starChart/views/talkView/talk_view_logic.dart +++ b/lib/talk/starChart/views/talkView/talk_view_logic.dart @@ -45,12 +45,12 @@ class TalkViewLogic extends BaseGetXController { final int minAudioBufferSize = 1; // 音频最小缓冲1帧 final int maxAudioBufferSize = 3; // 音频最大缓冲3帧 int audioBufferSize = 2; // 音频默认缓冲2帧 - + bool _isFirstAudioFrame = true; // 是否是第一帧 // 添加开始时间记录 int _startTime = 0; // 开始播放时间戳 int _startAudioTime = 0; // 开始播放时间戳 bool _isFirstFrame = true; // 是否是第一帧 - bool _isFirstAudioFrame = true; // 是否是第一帧 + // 定义音频帧缓冲和发送函数 final List _bufferedAudioFrames = []; @@ -106,6 +106,24 @@ class TalkViewLogic extends BaseGetXController { // 判断数据类型，进行分发处理 switch (contentType) { case TalkData_ContentTypeE.G711: + // // 第一帧到达时记录开始时间 + if (_isFirstAudioFrame) { + _startAudioTime = currentTime; + _isFirstAudioFrame = false; + } + + // 计算音频延迟 + final expectedTime = _startAudioTime + talkData.durationMs; + final audioDelay = currentTime - expectedTime; + + // 如果延迟太大，清空缓冲区并直接播放 + if (audioDelay > 500) { + state.audioBuffer.clear(); + if (state.isOpenVoice.value) { + _playAudioFrames(); + } + return; + } if (state.audioBuffer.length >= audioBufferSize) { state.audioBuffer.removeAt(0); // 丢弃最旧的数据 } @@ -118,7 +136,7 @@ class TalkViewLogic extends BaseGetXController { if (_isFirstFrame) { _startTime = currentTime; _isFirstFrame = false; - AppLog.log('第一帧帧的时间戳：${talkData.durationMs}'); + // AppLog.log('第一帧帧的时间戳：${talkData.durationMs}'); } // AppLog.log('其他帧的时间戳：${talkData.durationMs}'); // 计算帧间间隔 @@ -366,19 +384,6 @@ class TalkViewLogic extends BaseGetXController { } } - // 获取手机联网token，根据锁设置里面获取的开锁时是否联网来判断是否调用这个接口 - Future _getLockNetToken() async { - final LockNetTokenEntity entity = await ApiRepository.to.getLockNetToken( - lockId: lockDetailState.keyInfos.value.lockId.toString()); - if (entity.errorCode!.codeIsSuccessful) { - lockDetailState.lockNetToken = entity.data!.token!.toString(); - AppLog.log('从服务器获取联网token:${lockDetailState.lockNetToken}'); - } else { - BuglyTool.uploadException( - message: '点击了需要联网开锁', detail: '点击了需要联网开锁 获取连网token失败', upload: true); - showToast('网络访问失败,请检查网络是否正常'.tr, something: () {}); - } - } /// 获取权限状态 Future getPermissionStatus() async { diff --git a/lib/talk/starChart/webView/h264_web_logic.dart b/lib/talk/starChart/webView/h264_web_logic.dart index 626dff04..4f5789c2 100644 --- a/lib/talk/starChart/webView/h264_web_logic.dart +++ b/lib/talk/starChart/webView/h264_web_logic.dart @@ -48,7 +48,7 @@ class H264WebViewLogic extends BaseGetXController { Timer? _mockDataTimer; int _startAudioTime = 0; // 开始播放时间戳 int audioBufferSize = 2; // 音频默认缓冲2帧 - + bool _isFirstAudioFrame = true; // 是否是第一帧 // 定义音频帧缓冲和发送函数 final List _bufferedAudioFrames = []; final Queue> _frameBuffer = Queue>(); @@ -131,6 +131,24 @@ class H264WebViewLogic extends BaseGetXController { // 判断数据类型，进行分发处理 switch (contentType) { case TalkData_ContentTypeE.G711: + // // 第一帧到达时记录开始时间 + if (_isFirstAudioFrame) { + _startAudioTime = currentTime; + _isFirstAudioFrame = false; + } + + // 计算音频延迟 + final expectedTime = _startAudioTime + talkData.durationMs; + final audioDelay = currentTime - expectedTime; + + // 如果延迟太大，清空缓冲区并直接播放 + if (audioDelay > 500) { + state.audioBuffer.clear(); + if (state.isOpenVoice.value) { + _playAudioFrames(); + } + return; + } if (state.audioBuffer.length >= audioBufferSize) { state.audioBuffer.removeAt(0); // 丢弃最旧的数据 } diff --git a/pubspec.yaml b/pubspec.yaml index 454bd944..331117fb 100755 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -127,7 +127,8 @@ dependencies: sdk: flutter aliyun_face_plugin: path: aliyun_face_plugin - + video_decode_plugin: + path: ../video_decode_plugin flutter_localizations: sdk: flutter