From 8478dd33d1456eedfb930f4df8f233c7929f7459 Mon Sep 17 00:00:00 2001 From: liyi Date: Tue, 22 Apr 2025 15:17:42 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E8=B0=83=E6=95=B4=E4=BC=98=E5=8C=96h264?= =?UTF-8?q?=E6=92=AD=E6=94=BE=E9=80=BB=E8=BE=91=E5=B9=B6=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=9F=B3=E9=A2=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../lockDetail/lockDetail_logic.dart | 3 +- .../lockMian/entity/lockListInfo_entity.dart | 12 ++ .../handle/impl/udp_echo_test_handler.dart | 3 +- .../handle/impl/udp_talk_accept_handler.dart | 23 ++- .../handle/impl/udp_talk_data_handler.dart | 17 ++- .../handle/impl/udp_talk_expect_handler.dart | 4 +- .../handle/impl/udp_talk_request_handler.dart | 37 +++-- .../handle/other/h264_frame_handler.dart | 11 +- .../handle/other/talk_data_model.dart | 9 ++ .../handle/other/talk_data_repository.dart | 9 +- .../handle/scp_message_base_handle.dart | 6 +- lib/talk/starChart/star_chart_manage.dart | 37 ++--- .../views/talkView/talk_view_logic.dart | 27 +--- .../starChart/webView/h264_web_logic.dart | 133 +++++++++++++++--- .../webView/h264_web_view_state.dart | 3 + 15 files changed, 244 insertions(+), 90 deletions(-) create mode 100644 lib/talk/starChart/handle/other/talk_data_model.dart diff --git a/lib/main/lockDetail/lockDetail/lockDetail_logic.dart b/lib/main/lockDetail/lockDetail/lockDetail_logic.dart index f3668a2c..c03a2753 100755 --- a/lib/main/lockDetail/lockDetail/lockDetail_logic.dart +++ b/lib/main/lockDetail/lockDetail/lockDetail_logic.dart @@ -774,9 +774,10 @@ class LockDetailLogic extends BaseGetXController { showToast('设备未配网'.tr); return; } + // 发送监控id StartChartManage().startCallRequestMessageTimer( - ToPeerId: StartChartManage().lockPeerId ?? ''); + ToPeerId: StartChartManage().lockNetworkInfo.peerId ?? ''); } else { showToast('猫眼设置为省电模式时无法进行监控，请在猫眼设置中切换为其他模式'.tr); } diff --git a/lib/main/lockMian/entity/lockListInfo_entity.dart b/lib/main/lockMian/entity/lockListInfo_entity.dart index 754891c1..b98bb328 100755 --- a/lib/main/lockMian/entity/lockListInfo_entity.dart +++ b/lib/main/lockMian/entity/lockListInfo_entity.dart @@ -374,6 +374,9 @@ class LockFeature { this.isNoSupportedBlueBroadcast, this.wifiLockType, this.wifi, + this.isH264, + this.isH265, + this.isMJpeg, }); LockFeature.fromJson(Map json) { @@ -391,6 +394,9 @@ class LockFeature { isNoSupportedBlueBroadcast = json['isNoSupportedBlueBroadcast']; wifiLockType = json['wifiLockType']; wifi = json['wifi']; + isH264 = json['isH264']; + isH265 = json['isH265']; + isMJpeg = json['isMJpeg']; } int? password; @@ -407,6 +413,9 @@ class LockFeature { int? isNoSupportedBlueBroadcast; int? wifiLockType; int? wifi; + int? isH264; + int? isH265; + int? isMJpeg; Map toJson() { final Map data = {}; @@ -424,6 +433,9 @@ class LockFeature { data['isNoSupportedBlueBroadcast'] = isNoSupportedBlueBroadcast; data['wifiLockType'] = wifiLockType; data['wifi'] = wifi; + data['isH264'] = isH264; + data['isH265'] = isH265; + data['isMJpeg'] = isMJpeg; return data; } } diff --git a/lib/talk/starChart/handle/impl/udp_echo_test_handler.dart b/lib/talk/starChart/handle/impl/udp_echo_test_handler.dart index beebeb75..cff8154f 100644 --- a/lib/talk/starChart/handle/impl/udp_echo_test_handler.dart +++ b/lib/talk/starChart/handle/impl/udp_echo_test_handler.dart @@ -4,6 +4,7 @@ import 'dart:typed_data'; import 'package:flutter_easyloading/flutter_easyloading.dart'; import 'package:get/get.dart'; import 'package:star_lock/talk/starChart/entity/scp_message.dart'; +import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; import 'package:star_lock/talk/starChart/handle/scp_message_base_handle.dart'; import 'package:star_lock/talk/starChart/handle/scp_message_handle.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; @@ -22,7 +23,7 @@ class UdpEchoTestHandler extends ScpMessageBaseHandle EasyLoading.showToast(scpMessage.Payload, duration: 2000.milliseconds); } else { talkDataRepository.addTalkData( - TalkData(content: payload, contentType: TalkData_ContentTypeE.Image)); + TalkDataModel(talkData: TalkData(content: payload, contentType: TalkData_ContentTypeE.Image))); } } diff --git a/lib/talk/starChart/handle/impl/udp_talk_accept_handler.dart b/lib/talk/starChart/handle/impl/udp_talk_accept_handler.dart index 5f71aafa..2605525c 100644 --- a/lib/talk/starChart/handle/impl/udp_talk_accept_handler.dart +++ b/lib/talk/starChart/handle/impl/udp_talk_accept_handler.dart @@ -4,6 +4,7 @@ import 'dart:typed_data'; import 'package:flutter_easyloading/flutter_easyloading.dart'; import 'package:flutter_pcm_sound/flutter_pcm_sound.dart'; import 'package:get/get.dart'; +import 'package:star_lock/main/lockMian/entity/lockListInfo_entity.dart'; import 'package:star_lock/talk/starChart/constant/message_type_constant.dart'; import 'package:star_lock/talk/starChart/constant/talk_status.dart'; import 'package:star_lock/talk/starChart/entity/scp_message.dart'; @@ -13,6 +14,7 @@ import 'package:star_lock/talk/starChart/proto/gateway_reset.pb.dart'; import 'package:star_lock/talk/starChart/proto/generic.pb.dart'; import 'package:star_lock/talk/starChart/proto/talk_accept.pb.dart'; import 'package:star_lock/talk/starChart/proto/talk_expect.pb.dart'; +import 'package:star_lock/tools/commonDataManage.dart'; import '../../star_chart_manage.dart'; @@ -77,7 +79,24 @@ class UdpTalkAcceptHandler extends ScpMessageBaseHandle } void _handleSendExpect() { - // 修改预期数据并启动发送预期数据定时器，在收到回复时停止 - startChartManage.sendImageVideoAndG711AudioTalkExpectData(); + final LockListInfoItemEntity currentKeyInfo = + CommonDataManage().currentKeyInfo; + final isH264 = currentKeyInfo.lockFeature?.isH264 == 1; + final isMJpeg = currentKeyInfo.lockFeature?.isMJpeg == 1; + + // 优先使用H264，其次是MJPEG + if (isH264) { + // 锁支持H264，发送H264视频和G711音频期望 + startChartManage.sendH264VideoAndG711AudioTalkExpectData(); + print('锁支持H264，发送H264视频格式期望数据'); + } else if (isMJpeg) { + // 锁只支持MJPEG，发送图像视频和G711音频期望 + startChartManage.sendImageVideoAndG711AudioTalkExpectData(); + print('锁不支持H264，支持MJPEG，发送MJPEG视频格式期望数据'); + } else { + // 默认使用图像视频 + startChartManage.sendImageVideoAndG711AudioTalkExpectData(); + print('锁不支持H264和MJPEG，默认发送图像视频格式期望数据'); + } } } diff --git a/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart b/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart index 0a9e7ab3..44705624 100644 --- a/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart +++ b/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart @@ -5,6 +5,7 @@ import 'package:star_lock/talk/starChart/constant/message_type_constant.dart'; import 'package:star_lock/talk/starChart/entity/scp_message.dart'; import 'package:star_lock/talk/starChart/handle/other/h264_frame_handler.dart'; import 'package:star_lock/talk/starChart/handle/other/packet_loss_statistics.dart'; +import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; import 'package:star_lock/talk/starChart/handle/scp_message_base_handle.dart'; import 'package:star_lock/talk/starChart/handle/scp_message_handle.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; @@ -63,7 +64,7 @@ class UdpTalkDataHandler extends ScpMessageBaseHandle int? messageId}) { // 获取统计信息 final stats = PacketLossStatistics().getStatistics(); - _asyncLog('丢包统计: $stats'); + // _asyncLog('丢包统计: $stats'); // _asyncLog( // '分包数据:messageId:$messageId [$spIndex/$spTotal] PayloadLength:$PayloadLength'); if (messageType == MessageTypeConstant.RealTimeData) { @@ -118,7 +119,7 @@ class UdpTalkDataHandler extends ScpMessageBaseHandle void _handleVideoH264(TalkData talkData) { final TalkDataH264Frame talkDataH264Frame = TalkDataH264Frame(); talkDataH264Frame.mergeFromBuffer(talkData.content); - frameHandler.handleFrame(talkDataH264Frame); + frameHandler.handleFrame(talkDataH264Frame, talkData); } /// 处理图片数据 @@ -127,7 +128,11 @@ class UdpTalkDataHandler extends ScpMessageBaseHandle await _processCompletePayload(Uint8List.fromList(talkData.content)); processCompletePayload.forEach((element) { talkData.content = element; - talkDataRepository.addTalkData(talkData); + talkDataRepository.addTalkData( + TalkDataModel( + talkData: talkData, + ), + ); }); } @@ -138,7 +143,11 @@ class UdpTalkDataHandler extends ScpMessageBaseHandle // // 转pcm数据 // List pcmBytes = G711().convertList(g711Data); // talkData.content = pcmBytes; - talkDataRepository.addTalkData(talkData); + talkDataRepository.addTalkData( + TalkDataModel( + talkData: talkData, + ), + ); } catch (e) { print('Error decoding G.711 to PCM: $e'); } diff --git a/lib/talk/starChart/handle/impl/udp_talk_expect_handler.dart b/lib/talk/starChart/handle/impl/udp_talk_expect_handler.dart index 20dc1d2f..d1c83e77 100644 --- a/lib/talk/starChart/handle/impl/udp_talk_expect_handler.dart +++ b/lib/talk/starChart/handle/impl/udp_talk_expect_handler.dart @@ -20,7 +20,7 @@ import '../../star_chart_manage.dart'; class UdpTalkExpectHandler extends ScpMessageBaseHandle implements ScpMessageHandler { - final TalkViewState talkViewState = Get.put(TalkViewLogic()).state; + // final TalkViewState talkViewState = Get.put(TalkViewLogic()).state; @override void handleReq(ScpMessage scpMessage) { @@ -40,7 +40,7 @@ class UdpTalkExpectHandler extends ScpMessageBaseHandle startChartManage.stopTalkExpectMessageTimer(); // 停止发送对讲请求 startChartManage.stopCallRequestMessageTimer(); - talkViewState.rotateAngle.value = talkExpectResp.rotate ?? 0; + // talkViewState.rotateAngle.value = talkExpectResp.rotate ?? 0; // 收到预期数据的应答后，代表建立了连接，启动通话保持的监听 // 启动通话保持监听定时器（用来判断如果x秒内没有收到通话保持则执行的操作）; talkePingOverTimeTimerManager.start(); diff --git a/lib/talk/starChart/handle/impl/udp_talk_request_handler.dart b/lib/talk/starChart/handle/impl/udp_talk_request_handler.dart index be2f4565..974c8f25 100644 --- a/lib/talk/starChart/handle/impl/udp_talk_request_handler.dart +++ b/lib/talk/starChart/handle/impl/udp_talk_request_handler.dart @@ -7,7 +7,9 @@ import 'package:flutter_local_notifications/flutter_local_notifications.dart'; import 'package:get/get.dart'; import 'package:star_lock/appRouters.dart'; import 'package:star_lock/app_settings/app_settings.dart'; +import 'package:star_lock/main/lockMian/entity/lockListInfo_entity.dart'; import 'package:star_lock/talk/starChart/constant/message_type_constant.dart'; +import 'package:star_lock/talk/starChart/constant/talk_constant.dart'; import 'package:star_lock/talk/starChart/constant/talk_status.dart'; import 'package:star_lock/talk/starChart/entity/scp_message.dart'; import 'package:star_lock/talk/starChart/handle/scp_message_base_handle.dart'; @@ -16,6 +18,7 @@ import 'package:star_lock/talk/starChart/proto/gateway_reset.pb.dart'; import 'package:star_lock/talk/starChart/proto/generic.pb.dart'; import 'package:star_lock/talk/starChart/proto/talk_expect.pb.dart'; import 'package:star_lock/talk/starChart/proto/talk_request.pb.dart'; +import 'package:star_lock/tools/commonDataManage.dart'; import 'package:star_lock/tools/push/xs_jPhush.dart'; import 'package:star_lock/tools/storage.dart'; import 'package:star_lock/translations/current_locale_tool.dart'; @@ -30,7 +33,6 @@ class UdpTalkRequestHandler extends ScpMessageBaseHandle @override void handleReq(ScpMessage scpMessage) async { - final currentTime = DateTime.now().millisecondsSinceEpoch; // 确保与上次请求间隔至少1秒 if (currentTime - _lastRequestTime < 1000) { @@ -105,18 +107,16 @@ class UdpTalkRequestHandler extends ScpMessageBaseHandle _showTalkRequestNotification(talkObjectName: talkObjectName); // 设置为等待接听状态 talkStatus.setPassiveCallWaitingAnswer(); - // 收到呼叫请求，跳转到接听页面 if (startChartManage - .getDefaultTalkExpect() - .videoType - .indexOf(VideoTypeE.H264) == - -1) { + .getDefaultTalkExpect() + .videoType + .contains(VideoTypeE.H264)) { Get.toNamed( - Routers.starChartTalkView, + Routers.h264WebView, ); } else { Get.toNamed( - Routers.h264WebView, + Routers.starChartTalkView, ); } } @@ -189,7 +189,24 @@ class UdpTalkRequestHandler extends ScpMessageBaseHandle } void _handleSendExpect() { - // 修改预期数据并启动发送预期数据定时器，在收到回复时停止 - startChartManage.sendOnlyImageVideoTalkExpectData(); + final LockListInfoItemEntity currentKeyInfo = + CommonDataManage().currentKeyInfo; + final isH264 = currentKeyInfo.lockFeature?.isH264 == 1; + final isMJpeg = currentKeyInfo.lockFeature?.isMJpeg == 1; + + // 优先使用H264，其次是MJPEG + if (isH264) { + // 锁支持H264，发送H264视频和G711音频期望 + startChartManage.sendOnlyH264VideoTalkExpectData(); + print('锁支持H264，发送H264视频格式期望数据'); + } else if (isMJpeg) { + // 锁只支持MJPEG，发送图像视频和G711音频期望 + startChartManage.sendOnlyImageVideoTalkExpectData(); + print('锁不支持H264，支持MJPEG，发送MJPEG视频格式期望数据'); + } else { + // 默认使用图像视频 + startChartManage.sendOnlyImageVideoTalkExpectData(); + print('锁不支持H264和MJPEG，默认发送图像视频格式期望数据'); + } } } diff --git a/lib/talk/starChart/handle/other/h264_frame_handler.dart b/lib/talk/starChart/handle/other/h264_frame_handler.dart index cd2f8bc2..903524da 100644 --- a/lib/talk/starChart/handle/other/h264_frame_handler.dart +++ b/lib/talk/starChart/handle/other/h264_frame_handler.dart @@ -3,15 +3,20 @@ import 'dart:typed_data'; import 'package:flutter/services.dart'; import 'package:star_lock/app_settings/app_settings.dart'; +import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; +import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; import '../../proto/talk_data_h264_frame.pb.dart'; class H264FrameHandler { + final void Function(TalkDataModel frameData) onCompleteFrame; - final void Function(List frameData) onCompleteFrame; + // 只记录最近一个I帧的序号 + int _lastProcessedIFrameSeq = -1; H264FrameHandler({required this.onCompleteFrame}); - void handleFrame(TalkDataH264Frame frame) { - onCompleteFrame(frame.frameData); + void handleFrame(TalkDataH264Frame frame, TalkData talkData) { + onCompleteFrame( + TalkDataModel(talkData: talkData, talkDataH264Frame: frame)); } } diff --git a/lib/talk/starChart/handle/other/talk_data_model.dart b/lib/talk/starChart/handle/other/talk_data_model.dart new file mode 100644 index 00000000..a84f1d3d --- /dev/null +++ b/lib/talk/starChart/handle/other/talk_data_model.dart @@ -0,0 +1,9 @@ +import 'package:star_lock/talk/starChart/proto/talk_data.pbserver.dart'; +import 'package:star_lock/talk/starChart/proto/talk_data_h264_frame.pb.dart'; + +class TalkDataModel { + TalkData? talkData; + TalkDataH264Frame? talkDataH264Frame; + + TalkDataModel({required this.talkData, this.talkDataH264Frame}); +} diff --git a/lib/talk/starChart/handle/other/talk_data_repository.dart b/lib/talk/starChart/handle/other/talk_data_repository.dart index 062835cb..587a481f 100644 --- a/lib/talk/starChart/handle/other/talk_data_repository.dart +++ b/lib/talk/starChart/handle/other/talk_data_repository.dart @@ -1,9 +1,10 @@ import 'dart:async'; +import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; class TalkDataRepository { TalkDataRepository._() { - _talkDataStreamController = StreamController.broadcast( + _talkDataStreamController = StreamController.broadcast( onListen: () { _isListening = true; }, @@ -18,13 +19,13 @@ class TalkDataRepository { static TalkDataRepository get instance => _instance; - late final StreamController _talkDataStreamController; + late final StreamController _talkDataStreamController; bool _isListening = false; // 直接返回原始流，不做转换 - Stream get talkDataStream => _talkDataStreamController.stream; + Stream get talkDataStream => _talkDataStreamController.stream; - void addTalkData(TalkData talkData) { + void addTalkData(TalkDataModel talkData) { if (_isListening) { _talkDataStreamController.add(talkData); } diff --git a/lib/talk/starChart/handle/scp_message_base_handle.dart b/lib/talk/starChart/handle/scp_message_base_handle.dart index 11328f46..495db676 100644 --- a/lib/talk/starChart/handle/scp_message_base_handle.dart +++ b/lib/talk/starChart/handle/scp_message_base_handle.dart @@ -15,6 +15,7 @@ import 'package:star_lock/talk/starChart/constant/payload_type_constant.dart'; import 'package:star_lock/talk/starChart/constant/udp_constant.dart'; import 'package:star_lock/talk/starChart/entity/scp_message.dart'; import 'package:star_lock/talk/starChart/handle/other/h264_frame_handler.dart'; +import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; import 'package:star_lock/talk/starChart/handle/other/talk_data_repository.dart'; import 'package:star_lock/talk/starChart/handle/other/talke_data_over_time_timer_manager.dart'; @@ -55,10 +56,10 @@ class ScpMessageBaseHandle { // 处理出完整帧数据后的回调 final H264FrameHandler frameHandler = - H264FrameHandler(onCompleteFrame: (frameData) { + H264FrameHandler(onCompleteFrame: (TalkDataModel talkDataModel) { // 处理完整的帧数据 TalkDataRepository.instance.addTalkData( - TalkData(contentType: TalkData_ContentTypeE.H264, content: frameData), + talkDataModel, ); }); @@ -71,6 +72,7 @@ class ScpMessageBaseHandle { messageId: scpMessage.MessageId!, ); } + // 回复失败消息 void replyErrorMessage(ScpMessage scpMessage) { startChartManage.sendGenericRespErrorMessage( diff --git a/lib/talk/starChart/star_chart_manage.dart b/lib/talk/starChart/star_chart_manage.dart index 7ffed8e5..a8f9ad5f 100644 --- a/lib/talk/starChart/star_chart_manage.dart +++ b/lib/talk/starChart/star_chart_manage.dart @@ -113,7 +113,7 @@ class StartChartManage { final int _maxPayloadSize = 8 * 1024; // 分包大小 // 默认通话的期望数据格式 - TalkExpectReq _defaultTalkExpect = TalkConstant.ImageExpect; + TalkExpectReq _defaultTalkExpect = TalkConstant.H264Expect; String relayPeerId = ''; // 中继peerId @@ -227,20 +227,6 @@ class StartChartManage { /// 设置数据接收回调 _onReceiveData(_udpSocket!, Get.context!); - - // //ToDo: 增加对讲调试、正式可删除 - // // 每秒重置数据速率 - // Timer.periodic(Duration(seconds: 1), (Timer t) { - // UdpTalkDataHandler().resetDataRates(); - // // 更新调试信息 - // Provider.of(Get.context!, listen: false) - // .updateDebugInfo( - // UdpTalkDataHandler().getLastRecvDataRate() ~/ 1024, // 转换为KB - // UdpTalkDataHandler().getLastRecvPacketCount(), - // UdpTalkDataHandler().getLastSendDataRate() ~/ 1024, // 转换为KB - // UdpTalkDataHandler().getLastSendPacketCount(), - // ); - // }); }).catchError((error) { _log(text: 'Failed to bind UDP socket: $error'); }); @@ -1145,7 +1131,7 @@ class StartChartManage { } void reSetDefaultTalkExpect() { - _defaultTalkExpect = TalkConstant.ImageExpect; + _defaultTalkExpect = TalkConstant.H264Expect; } TalkExpectReq getDefaultTalkExpect() { @@ -1163,12 +1149,27 @@ class StartChartManage { } /// 修改预期接收到的数据 - void sendImageVideoAndG711AudioTalkExpectData() { - final talkExpectReq = TalkConstant.ImageExpect; + void sendOnlyH264VideoTalkExpectData() { + final talkExpectReq = TalkExpectReq( + videoType: [VideoTypeE.H264], + audioType: [], + ); changeTalkExpectDataTypeAndReStartTalkExpectMessageTimer( talkExpect: talkExpectReq); } + /// 修改预期接收到的数据 + void sendImageVideoAndG711AudioTalkExpectData() { + changeTalkExpectDataTypeAndReStartTalkExpectMessageTimer( + talkExpect: TalkConstant.ImageExpect); + } + + /// 修改预期接收到的数据 + void sendH264VideoAndG711AudioTalkExpectData() { + changeTalkExpectDataTypeAndReStartTalkExpectMessageTimer( + talkExpect: TalkConstant.H264Expect); + } + /// 发送远程开锁 void sendRemoteUnLockMessage({ required String bluetoothDeviceName, diff --git a/lib/talk/starChart/views/talkView/talk_view_logic.dart b/lib/talk/starChart/views/talkView/talk_view_logic.dart index 779e44e9..a4a850ba 100644 --- a/lib/talk/starChart/views/talkView/talk_view_logic.dart +++ b/lib/talk/starChart/views/talkView/talk_view_logic.dart @@ -23,6 +23,7 @@ import 'package:star_lock/main/lockMian/entity/lockListInfo_entity.dart'; import 'package:star_lock/network/api_repository.dart'; import 'package:star_lock/talk/call/g711.dart'; import 'package:star_lock/talk/starChart/constant/talk_status.dart'; +import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; import 'package:star_lock/talk/starChart/proto/talk_expect.pb.dart'; import 'package:star_lock/talk/starChart/star_chart_manage.dart'; @@ -96,31 +97,15 @@ class TalkViewLogic extends BaseGetXController { // 监听音视频数据流 void _startListenTalkData() { - state.talkDataRepository.talkDataStream.listen((TalkData talkData) async { - final contentType = talkData.contentType; + state.talkDataRepository.talkDataStream + .listen((TalkDataModel talkDataModel) async { + final talkData = talkDataModel.talkData; + final contentType = talkData!.contentType; final currentTime = DateTime.now().millisecondsSinceEpoch; // 判断数据类型，进行分发处理 switch (contentType) { case TalkData_ContentTypeE.G711: - // // 第一帧到达时记录开始时间 - // if (_isFirstAudioFrame) { - // _startAudioTime = currentTime; - // _isFirstAudioFrame = false; - // } - - // 计算音频延迟 - final expectedTime = _startAudioTime + talkData.durationMs; - final audioDelay = currentTime - expectedTime; - - // 如果延迟太大，清空缓冲区并直接播放 - if (audioDelay > 500) { - state.audioBuffer.clear(); - if (state.isOpenVoice.value) { - _playAudioFrames(); - } - return; - } if (state.audioBuffer.length >= audioBufferSize) { state.audioBuffer.removeAt(0); // 丢弃最旧的数据 } @@ -195,7 +180,6 @@ class TalkViewLogic extends BaseGetXController { // _frameCount = 0; // _lastFpsUpdateTime = currentTime; // } - } else { // AppLog.log('⚠️ 帧未找到缓存 - Key: $cacheKey'); state.videoBuffer.removeAt(oldestIndex); // 移除无法播放的帧 @@ -497,7 +481,6 @@ class TalkViewLogic extends BaseGetXController { _initAudioRecorder(); requestPermissions(); - } @override diff --git a/lib/talk/starChart/webView/h264_web_logic.dart b/lib/talk/starChart/webView/h264_web_logic.dart index fdbaad3f..626dff04 100644 --- a/lib/talk/starChart/webView/h264_web_logic.dart +++ b/lib/talk/starChart/webView/h264_web_logic.dart @@ -25,7 +25,9 @@ import 'package:star_lock/network/api_repository.dart'; import 'package:star_lock/talk/call/g711.dart'; import 'package:star_lock/talk/starChart/constant/talk_status.dart'; import 'package:star_lock/talk/starChart/handle/other/packet_loss_statistics.dart'; +import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; +import 'package:star_lock/talk/starChart/proto/talk_data_h264_frame.pb.dart'; import 'package:star_lock/talk/starChart/proto/talk_expect.pb.dart'; import 'package:star_lock/talk/starChart/star_chart_manage.dart'; import 'package:star_lock/talk/starChart/views/talkView/talk_view_state.dart'; @@ -44,6 +46,9 @@ class H264WebViewLogic extends BaseGetXController { // 添加模拟数据相关变量 static const int CHUNK_SIZE = 4096; Timer? _mockDataTimer; + int _startAudioTime = 0; // 开始播放时间戳 + int audioBufferSize = 2; // 音频默认缓冲2帧 + // 定义音频帧缓冲和发送函数 final List _bufferedAudioFrames = []; final Queue> _frameBuffer = Queue>(); @@ -51,7 +56,6 @@ class H264WebViewLogic extends BaseGetXController { @override void onInit() { - super.onInit(); // 初始化 WebView 控制器 state.webViewController = WebViewController() ..setJavaScriptMode(JavaScriptMode.unrestricted) @@ -63,18 +67,40 @@ class H264WebViewLogic extends BaseGetXController { }, ); - state.isShowLoading.value = true; - // 加载本地 HTML - _loadLocalHtml(); + super.onInit(); // 创建流数据监听 _createFramesStreamListen(); - // playLocalTestVideo(); _startListenTalkStatus(); state.talkStatus.value = state.startChartTalkStatus.status; // 初始化音频播放器 _initFlutterPcmSound(); + // 初始化录音控制器 _initAudioRecorder(); + + // 加载本地 HTML + _loadLocalHtml(); + + // playLocalTestVideo(); + + requestPermissions(); + } + + Future requestPermissions() async { + // 申请存储权限 + var storageStatus = await Permission.storage.request(); + // 申请录音权限 + var microphoneStatus = await Permission.microphone.request(); + + if (storageStatus.isGranted && microphoneStatus.isGranted) { + print("Permissions granted"); + } else { + print("Permissions denied"); + // 如果权限被拒绝，可以提示用户或跳转到设置页面 + if (await Permission.storage.isPermanentlyDenied) { + openAppSettings(); // 跳转到应用设置页面 + } + } } /// 初始化音频录制器 @@ -96,16 +122,37 @@ class H264WebViewLogic extends BaseGetXController { } void _createFramesStreamListen() async { - state.talkDataRepository.talkDataStream.listen((TalkData event) async { - // 添加新帧到缓冲区 - _frameBuffer.add(event.content); + state.talkDataRepository.talkDataStream + .listen((TalkDataModel talkDataModel) async { + final talkData = talkDataModel.talkData; + final contentType = talkData!.contentType; + final currentTime = DateTime.now().millisecondsSinceEpoch; - // 当缓冲区超过最大容量时,发送最早的帧并移除 - while (_frameBuffer.length > FRAME_BUFFER_SIZE) { - if (_frameBuffer.isNotEmpty) { - final frame = _frameBuffer.removeFirst(); - await _sendBufferedData(frame); - } + // 判断数据类型，进行分发处理 + switch (contentType) { + case TalkData_ContentTypeE.G711: + if (state.audioBuffer.length >= audioBufferSize) { + state.audioBuffer.removeAt(0); // 丢弃最旧的数据 + } + state.audioBuffer.add(talkData); // 添加新数据 + // 添加音频播放逻辑，与视频类似 + _playAudioFrames(); + break; + case TalkData_ContentTypeE.H264: + // // 添加新帧到缓冲区 + _frameBuffer.add(talkData.content); + + // 当缓冲区超过最大容量时,发送最早的帧并移除 + while (_frameBuffer.length > FRAME_BUFFER_SIZE) { + if (_frameBuffer.isNotEmpty) { + final frame = _frameBuffer.removeFirst(); + await _sendBufferedData(frame); + } + if (state.isShowLoading.isTrue) { + state.isShowLoading.value = false; + } + } + break; } }); } @@ -134,6 +181,51 @@ class H264WebViewLogic extends BaseGetXController { // } // } + // 新增：音频帧播放逻辑 + void _playAudioFrames() { + // 如果缓冲区为空或未达到目标大小，不进行播放 + // 音频缓冲区要求更小，以减少延迟 + if (state.audioBuffer.isEmpty || + state.audioBuffer.length < audioBufferSize) { + return; + } + + // 找出时间戳最小的音频帧 + TalkData? oldestFrame; + int oldestIndex = -1; + for (int i = 0; i < state.audioBuffer.length; i++) { + if (oldestFrame == null || + state.audioBuffer[i].durationMs < oldestFrame.durationMs) { + oldestFrame = state.audioBuffer[i]; + oldestIndex = i; + } + } + + // 确保找到了有效帧 + if (oldestFrame != null && oldestIndex != -1) { + if (state.isOpenVoice.value) { + // 播放音频 + _playAudioData(oldestFrame); + } + state.audioBuffer.removeAt(oldestIndex); + } + } + + /// 播放音频数据 + void _playAudioData(TalkData talkData) async { + if (state.isOpenVoice.value) { + final list = + G711().decodeAndDenoise(talkData.content, true, 8000, 300, 150); + // // 将 PCM 数据转换为 PcmArrayInt16 + final PcmArrayInt16 fromList = PcmArrayInt16.fromList(list); + FlutterPcmSound.feed(fromList); + if (!state.isPlaying.value) { + FlutterPcmSound.play(); + state.isPlaying.value = true; + } + } + } + /// 加载html文件 Future _loadLocalHtml() async { // 加载 HTML 文件内容 @@ -186,10 +278,10 @@ class H264WebViewLogic extends BaseGetXController { Timer.periodic(const Duration(seconds: 1), (Timer t) { if (state.isShowLoading.isFalse) { state.oneMinuteTime.value++; - if (state.oneMinuteTime.value >= 60) { - t.cancel(); // 取消定时器 - state.oneMinuteTime.value = 0; - } + // if (state.oneMinuteTime.value >= 60) { + // t.cancel(); // 取消定时器 + // state.oneMinuteTime.value = 0; + // } } }); break; @@ -321,7 +413,7 @@ class H264WebViewLogic extends BaseGetXController { } // 首先应用固定增益提升基础音量 - List amplifiedFrame = _applyGain(frame, 1.6); + List amplifiedFrame = _applyGain(frame, 1.8); // 编码为G711数据 List encodedData = G711Tool.encode(amplifiedFrame, 0); // 0表示A-law _bufferedAudioFrames.addAll(encodedData); @@ -409,7 +501,7 @@ class H264WebViewLogic extends BaseGetXController { }); final LockSetInfoEntity lockSetInfoEntity = - await ApiRepository.to.getLockSettingInfoData( + await ApiRepository.to.getLockSettingInfoData( lockId: lockId.toString(), ); if (lockSetInfoEntity.errorCode!.codeIsSuccessful) { @@ -427,7 +519,6 @@ class H264WebViewLogic extends BaseGetXController { } } - @override void dispose() { // _mockDataTimer?.cancel(); diff --git a/lib/talk/starChart/webView/h264_web_view_state.dart b/lib/talk/starChart/webView/h264_web_view_state.dart index 2ae11041..671e805c 100644 --- a/lib/talk/starChart/webView/h264_web_view_state.dart +++ b/lib/talk/starChart/webView/h264_web_view_state.dart @@ -5,6 +5,7 @@ import 'package:flutter_voice_processor/flutter_voice_processor.dart'; import 'package:get/get.dart'; import 'package:star_lock/talk/starChart/constant/talk_status.dart'; import 'package:star_lock/talk/starChart/handle/other/talk_data_repository.dart'; +import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; import 'package:star_lock/talk/starChart/status/star_chart_talk_status.dart'; import 'package:star_lock/talk/starChart/views/talkView/talk_view_state.dart'; import 'package:webview_flutter/webview_flutter.dart'; @@ -49,4 +50,6 @@ class H264WebViewState { RxInt rotateAngle = 0.obs; // 旋转角度（以弧度为单位） RxBool hasAudioData = false.obs; // 是否有音频数据 RxInt lastAudioTimestamp = 0.obs; // 最后接收到的音频数据的时间戳 + List audioBuffer = [].obs; + RxBool isPlaying = false.obs; // 是否开始播放 }