From e850179c923228063e355b9199e885841ca67f10 Mon Sep 17 00:00:00 2001 From: liyi Date: Mon, 30 Dec 2024 11:53:42 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E5=A2=9E=E5=8A=A0=E5=BD=95=E9=9F=B3?= =?UTF-8?q?=E5=8F=91=E9=80=81=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../startChart/command/message_command.dart | 8 +- .../handle/impl/udp_go_online_handler.dart | 4 +- .../handle/impl/udp_talk_ping_handler.dart | 1 + .../handle/scp_message_base_handle.dart | 11 +- lib/talk/startChart/start_chart_manage.dart | 66 ++++++-- .../views/talkView/talk_view_logic.dart | 155 ++++++++++++++++-- .../views/talkView/talk_view_page.dart | 82 +++++---- .../views/talkView/talk_view_state.dart | 18 +- 8 files changed, 270 insertions(+), 75 deletions(-) diff --git a/lib/talk/startChart/command/message_command.dart b/lib/talk/startChart/command/message_command.dart index 86184110..82b8ade6 100644 --- a/lib/talk/startChart/command/message_command.dart +++ b/lib/talk/startChart/command/message_command.dart @@ -270,10 +270,12 @@ class MessageCommand { static List talkDataMessage({ required String FromPeerId, required String ToPeerId, - required TalkData talkData, int? MessageId, + List? payload, + int? SpTotal, + int? SpIndex, }) { - final payload = talkData.writeToBuffer(); + // final payload = talkData.writeToBuffer(); ScpMessage message = ScpMessage( ProtocolFlag: ProtocolFlagConstant.scp01, MessageType: MessageTypeConstant.RealTimeData, @@ -283,7 +285,7 @@ class MessageCommand { FromPeerId: FromPeerId, ToPeerId: ToPeerId, Payload: payload, - PayloadCRC: calculationCrc(payload), + PayloadCRC: calculationCrc(Uint8List.fromList(payload!)), PayloadLength: payload.length, PayloadType: PayloadTypeConstant.talkData, ); diff --git a/lib/talk/startChart/handle/impl/udp_go_online_handler.dart b/lib/talk/startChart/handle/impl/udp_go_online_handler.dart index 7899adc9..f3e9dd69 100644 --- a/lib/talk/startChart/handle/impl/udp_go_online_handler.dart +++ b/lib/talk/startChart/handle/impl/udp_go_online_handler.dart @@ -25,7 +25,7 @@ class UdpGoOnlineHandler extends ScpMessageBaseHandle startChartManage.isOnlineStartChartServer = true; // 上线成功，停止重发 startChartManage.stopReStartOnlineStartChartServer(); - log(text: '星图登录成功'); + log(text: '星图登录成功,PeerID:${scpMessage.ToPeerId}'); } else { // 上线失败，重发 startChartManage.reStartOnlineStartChartServer(); @@ -42,7 +42,7 @@ class UdpGoOnlineHandler extends ScpMessageBaseHandle deserializePayload( {required int payloadType, required int messageType, - required List byte, + required List byte, int? offset, int? PayloadLength, int? spTotal, diff --git a/lib/talk/startChart/handle/impl/udp_talk_ping_handler.dart b/lib/talk/startChart/handle/impl/udp_talk_ping_handler.dart index 95c9f243..375cb911 100644 --- a/lib/talk/startChart/handle/impl/udp_talk_ping_handler.dart +++ b/lib/talk/startChart/handle/impl/udp_talk_ping_handler.dart @@ -14,6 +14,7 @@ class UdpTalkPingHandler extends ScpMessageBaseHandle @override void handleReq(ScpMessage scpMessage) { // TODO: 收到通话保持请求 + replySuccessMessage(scpMessage); } @override diff --git a/lib/talk/startChart/handle/scp_message_base_handle.dart b/lib/talk/startChart/handle/scp_message_base_handle.dart index 5a0aabcd..1db03a29 100644 --- a/lib/talk/startChart/handle/scp_message_base_handle.dart +++ b/lib/talk/startChart/handle/scp_message_base_handle.dart @@ -38,17 +38,17 @@ class ScpMessageBaseHandle { // 通话请求超时未处理监听定时器管理 final talkeRequestOverTimeTimerManager = OverTimeTimerManager( - timeoutInSeconds: 30, + timeoutInSeconds: 8, ); // 通话保持超时监听定时器管理 final talkePingOverTimeTimerManager = OverTimeTimerManager( - timeoutInSeconds: 260, + timeoutInSeconds: 5, ); // 通话数据超时定时器 final talkDataOverTimeTimerManager = OverTimeTimerManager( - timeoutInSeconds: 260, + timeoutInSeconds: 30, ); // 回复成功消息 @@ -64,8 +64,9 @@ class ScpMessageBaseHandle { if (genericResp == null) return false; final code = genericResp.code; final message = genericResp.message; - return code == UdpConstant.genericRespSuccessCode && - message == UdpConstant.genericRespSuccessMsg; + return (code == UdpConstant.genericRespSuccessCode || code == null) && + message.toLowerCase() == + UdpConstant.genericRespSuccessMsg.toLowerCase(); } void log({required String text}) { diff --git a/lib/talk/startChart/start_chart_manage.dart b/lib/talk/startChart/start_chart_manage.dart index e5087f6d..f587056d 100644 --- a/lib/talk/startChart/start_chart_manage.dart +++ b/lib/talk/startChart/start_chart_manage.dart @@ -92,7 +92,7 @@ class StartChartManage { ); // 默认通话数据 - TalkData defaultTalkData = TalkData(); + TalkData _defaultTalkData = TalkData(); String relayPeerId = ''; // 中继peerId @@ -252,14 +252,44 @@ class StartChartManage { // 发送对讲数据 Future sendTalkDataMessage({required TalkData talkData}) async { - // 组装上线消息 - final message = MessageCommand.talkDataMessage( - FromPeerId: FromPeerId, - ToPeerId: ToPeerId, - talkData: talkData, - MessageId: MessageCommand.getNextMessageId(ToPeerId, increment: true), - ); - await _sendMessage(message: message); + final List payload = talkData.content; + // 计算需要分多少个包发送 + final int totalPackets = (payload.length / _maxPayloadSize).ceil(); + // 循环遍历 + for (int i = 0; i < totalPackets; i++) { + int start = i * _maxPayloadSize; + int end = (i + 1) * _maxPayloadSize; + if (end > payload.length) { + end = payload.length; + } + // 取到分包数据 + List packet = payload.sublist(start, end); + + // 分包数据不递增messageID + final messageId = + MessageCommand.getNextMessageId(ToPeerId, increment: false); + // 组装分包数据 + final message = MessageCommand.talkDataMessage( + ToPeerId: ToPeerId, + FromPeerId: FromPeerId, + payload: packet, + SpTotal: totalPackets, + SpIndex: i + 1, + MessageId: messageId, + ); + // 发送消息 + await _sendMessage(message: message); + } + // 分包发送完了递增一下id + MessageCommand.getNextMessageId(ToPeerId); + // + // final message = MessageCommand.talkDataMessage( + // FromPeerId: FromPeerId, + // ToPeerId: ToPeerId, + // talkData: talkData, + // MessageId: MessageCommand.getNextMessageId(ToPeerId, increment: true), + // ); + // await _sendMessage(message: message); } // 发送心跳包消息 @@ -350,7 +380,7 @@ class StartChartManage { await _sendMessage(message: message); // 设置状态为等待接听 talkStatus.setWaitingAnswer(); - _log(text: '发送同意接听消息'); + // _log(text: '发送同意接听消息'); } // 发送拒绝接听消息 @@ -381,6 +411,7 @@ class StartChartManage { MessageId: MessageCommand.getNextMessageId(ToPeerId, increment: true), ); await _sendMessage(message: message); + // _log(text: '发送预期数据：${talkExpect}'); } // 回复成功消息 @@ -420,6 +451,7 @@ class StartChartManage { MessageId: MessageCommand.getNextMessageId(ToPeerId, increment: true), ); await _sendMessage(message: message); + _log(text: '发送通话保持'); } // 发送通话中挂断消息 @@ -892,13 +924,13 @@ class StartChartManage { current++; List frameData = byteData.sublist(start, end); if (frameData.length == 0) timer.cancel(); - defaultTalkData = TalkData( + _defaultTalkData = TalkData( content: frameData, contentType: TalkData_ContentTypeE.H264, ); start = end; - // 发送童话数据 - sendTalkDataMessage(talkData: defaultTalkData); + // 发送通话数据 + sendTalkDataMessage(talkData: _defaultTalkData); }, ); } @@ -991,7 +1023,15 @@ class StartChartManage { stopHeartbeat(); stopReStartOnlineStartChartServer(); stopTalkDataTimer(); + _resetData(); // await Storage.removerRelayInfo(); // await Storage.removerStarChartRegisterNodeInfo(); } + + void _resetData() { + _defaultTalkExpect = TalkExpectReq( + videoType: [VideoTypeE.IMAGE], + audioType: [AudioTypeE.G711], + ); + } } diff --git a/lib/talk/startChart/views/talkView/talk_view_logic.dart b/lib/talk/startChart/views/talkView/talk_view_logic.dart index edd759ea..53016b02 100644 --- a/lib/talk/startChart/views/talkView/talk_view_logic.dart +++ b/lib/talk/startChart/views/talkView/talk_view_logic.dart @@ -7,6 +7,8 @@ import 'package:flutter/services.dart'; import 'package:flutter_easyloading/flutter_easyloading.dart'; import 'package:flutter_pcm_sound/flutter_pcm_sound.dart'; import 'package:flutter_screen_recording/flutter_screen_recording.dart'; + +import 'package:flutter_voice_processor/flutter_voice_processor.dart'; import 'package:gallery_saver/gallery_saver.dart'; import 'package:get/get.dart'; @@ -31,14 +33,12 @@ class TalkViewLogic extends BaseGetXController { Timer? _syncTimer; int _startTime = 0; final int bufferSize = 20; // 缓冲区大小（以帧为单位） - final List frameTimestamps = []; + final List frameTimestamps = []; // 帧时间戳用于计算 FPS int frameIntervalMs = 45; // 初始帧间隔设置为45毫秒（约22FPS） int minFrameIntervalMs = 30; // 最小帧间隔（约33 FPS） int maxFrameIntervalMs = 100; // 最大帧间隔（约10 FPS） - /// 收到Talk发送的状态 - StreamSubscription? _getTalkStatusRefreshUIEvent; - + /// 初始化音频播放器 void _initFlutterPcmSound() { const int sampleRate = 44100; FlutterPcmSound.setLogLevel(LogLevel.verbose); @@ -129,6 +129,7 @@ class TalkViewLogic extends BaseGetXController { }); } + /// 播放音频数据 void _playAudioData(TalkData talkData) { // 将 PCM 数据转换为 PcmArrayInt16 final PcmArrayInt16 fromList = PcmArrayInt16.fromList(talkData.content); @@ -139,6 +140,7 @@ class TalkViewLogic extends BaseGetXController { } } + /// 播放视频数据 void _playVideoData(TalkData talkData) { state.listData.value = Uint8List.fromList(talkData.content); } @@ -263,7 +265,9 @@ class TalkViewLogic extends BaseGetXController { /// 提示网络状态 void showNetworkStatus(String message) { - if (state.alertCount.value < 3 && !EasyLoading.isShow) { + // 如果提示次数未达到最大值且 EasyLoading 未显示，则显示提示 + if (state.alertCount.value < state.maxAlertNumber.value && + !EasyLoading.isShow) { showToast(message); state.alertCount++; } @@ -279,6 +283,7 @@ class TalkViewLogic extends BaseGetXController { /// 开门 udpOpenDoorAction(List list) async {} + /// 获取权限状态 Future getPermissionStatus() async { final Permission permission = Permission.microphone; //granted 通过，denied 被拒绝，permanentlyDenied 拒绝且不在提示 @@ -312,17 +317,22 @@ class TalkViewLogic extends BaseGetXController { void onInit() { super.onInit(); - // 监听音视频数据流 + // 启动监听音视频数据流 _startListenTalkData(); - // 监听对讲状态 + // 启动监听对讲状态 _startListenTalkStatus(); // 在没有监听成功之前赋值一遍状态 // *** 由于页面会在状态变化之后才会初始化，导致识别不到最新的状态，在这里手动赋值 *** state.talkStatus.value = state.startChartTalkStatus.status; + // 初始化音频播放器 _initFlutterPcmSound(); + // 启动播放定时器 _startPlayback(); + + // 初始化录音控制器 + _initAudioRecorder(); } @override @@ -404,7 +414,7 @@ class TalkViewLogic extends BaseGetXController { bool started = await FlutterScreenRecording.startRecordScreenAndAudio("Recording"); if (started) { - state.isRecording.value = true; + state.isRecordingScreen.value = true; } } @@ -412,13 +422,136 @@ class TalkViewLogic extends BaseGetXController { Future stopRecording() async { String path = await FlutterScreenRecording.stopRecordScreen; if (path != null) { - state.isRecording.value = false; + state.isRecordingScreen.value = false; // 保存录制的视频到相册 - await GallerySaver.saveVideo(path).then((bool? success) {}); + // await GallerySaver.saveVideo(path).then((bool? success) {}); + // 将截图保存到相册 + await ImageGallerySaver.saveFile(path); showToast('录屏已保存到相册'.tr); } else { - state.isRecording.value = false; + state.isRecordingScreen.value = false; print("Recording failed"); } } + + /// 初始化音频录制器 + void _initAudioRecorder() { + state.voiceProcessor = VoiceProcessor.instance; + } + + //开始录音 + Future startProcessingAudio() async { + // 增加录音帧监听器和错误监听器 + state.voiceProcessor?.addFrameListener(_onFrame); + state.voiceProcessor?.addErrorListener(_onError); + try { + if (await state.voiceProcessor?.hasRecordAudioPermission() ?? false) { + await state.voiceProcessor?.start(state.frameLength, state.sampleRate); + final bool? isRecording = await state.voiceProcessor?.isRecording(); + state.isRecordingAudio.value = isRecording!; + state.startRecordingAudioTime.value = DateTime.now(); + } else { + // state.errorMessage.value = 'Recording permission not granted'; + } + } on PlatformException catch (ex) { + // state.errorMessage.value = 'Failed to start recorder: $ex'; + } + } + + /// 停止录音 + Future stopProcessingAudio() async { + try { + await state.voiceProcessor?.stop(); + state.voiceProcessor?.removeFrameListener(_onFrame); + state.udpSendDataFrameNumber = 0; + // 记录结束时间 + state.endRecordingAudioTime.value = DateTime.now(); + + // 计算录音的持续时间 + final duration = state.endRecordingAudioTime.value! + .difference(state.startRecordingAudioTime.value!); + + state.recordingAudioTime.value = duration.inSeconds; + } on PlatformException catch (ex) { + // state.errorMessage.value = 'Failed to stop recorder: $ex'; + } finally { + final bool? isRecording = await state.voiceProcessor?.isRecording(); + state.isRecordingAudio.value = isRecording!; + } + } + + Future _onFrame(List frame) async { + state.recordingAudioAllFrames.add(frame); // 将帧添加到状态变量中 + // final List concatenatedFrames = + // _concatenateFrames(state.recordingAudioAllFrames); // 连接所有帧 + final List pcmBytes = _listLinearToULaw(frame); + // 发送音频数据 + StartChartManage().sendTalkDataMessage( + talkData: TalkData( + content: pcmBytes, + contentType: TalkData_ContentTypeE.G711, + durationMs: DateTime.now().millisecondsSinceEpoch - + state.startRecordingAudioTime.value.millisecondsSinceEpoch, + ), + ); + } + + void _onError(VoiceProcessorException error) { + // state.errorMessage.value = error.message!; + AppLog.log(error.message!); + } + +// 拿到的音频转化成pcm + List _listLinearToULaw(List pcmList) { + final List uLawList = []; + for (int pcmVal in pcmList) { + final int uLawVal = _linearToULaw(pcmVal); + uLawList.add(uLawVal); + } + return uLawList; + } + + // 拿到的音频转化成pcm + int _linearToULaw(int pcmVal) { + int mask; + int seg; + int uval; + + if (pcmVal < 0) { + pcmVal = 0x84 - pcmVal; + mask = 0x7F; + } else { + pcmVal += 0x84; + mask = 0xFF; + } + + seg = search(pcmVal); + if (seg >= 8) { + return 0x7F ^ mask; + } else { + uval = seg << 4; + uval |= (pcmVal >> (seg + 3)) & 0xF; + return uval ^ mask; + } + } + + int search(int val) { + final List table = [ + 0xFF, + 0x1FF, + 0x3FF, + 0x7FF, + 0xFFF, + 0x1FFF, + 0x3FFF, + 0x7FFF + ]; + const int size = 8; + for (int i = 0; i < size; i++) { + if (val <= table[i]) { + return i; + } + } + return size; + } } diff --git a/lib/talk/startChart/views/talkView/talk_view_page.dart b/lib/talk/startChart/views/talkView/talk_view_page.dart index 879dcb0a..2ab9de32 100644 --- a/lib/talk/startChart/views/talkView/talk_view_page.dart +++ b/lib/talk/startChart/views/talkView/talk_view_page.dart @@ -5,17 +5,14 @@ import 'dart:io'; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter/services.dart'; -import 'package:flutter_screen_recording/flutter_screen_recording.dart'; + import 'package:flutter_screenutil/flutter_screenutil.dart'; -import 'package:gallery_saver/gallery_saver.dart'; + import 'package:get/get.dart'; -import 'package:image_gallery_saver/image_gallery_saver.dart'; -import 'package:path_provider/path_provider.dart'; -import 'package:star_lock/app_settings/app_settings.dart'; -import 'package:star_lock/main/lockDetail/realTimePicture/realTimePicture_state.dart'; + import 'package:star_lock/talk/call/callTalk.dart'; import 'package:star_lock/talk/startChart/constant/talk_status.dart'; -import 'package:star_lock/talk/startChart/start_chart_talk_status.dart'; + import 'package:star_lock/talk/startChart/views/talkView/talk_view_logic.dart'; import 'package:star_lock/talk/startChart/views/talkView/talk_view_state.dart'; @@ -75,24 +72,30 @@ class _TalkViewPageState extends State height: ScreenUtil().screenHeight, fit: BoxFit.cover, ) - : PopScope( - canPop: false, - child: RepaintBoundary( - key: state.globalKey, - child: Image.memory( - state.listData.value, - gaplessPlayback: true, - width: 1.sw, - height: 1.sh, - fit: BoxFit.cover, - filterQuality: FilterQuality.high, - errorBuilder: ( - BuildContext context, - Object error, - StackTrace? stackTrace, - ) { - return Container(color: Colors.transparent); - }, + : Container( + decoration: state.isRecordingScreen.value + ? BoxDecoration( + border: Border.all(color: Colors.red, width: 1)) + : BoxDecoration(), + child: PopScope( + canPop: false, + child: RepaintBoundary( + key: state.globalKey, + child: Image.memory( + state.listData.value, + gaplessPlayback: true, + width: 1.sw, + height: 1.sh, + fit: BoxFit.cover, + filterQuality: FilterQuality.high, + errorBuilder: ( + BuildContext context, + Object error, + StackTrace? stackTrace, + ) { + return Container(color: Colors.transparent); + }, + ), ), ), ), @@ -151,7 +154,10 @@ class _TalkViewPageState extends State // 打开关闭声音 GestureDetector( onTap: () { - logic.updateTalkExpect(); + if (state.talkStatus.value == TalkStatus.duringCall) { + // 打开关闭声音 + logic.updateTalkExpect(); + } }, child: Container( width: 50.w, @@ -171,7 +177,9 @@ class _TalkViewPageState extends State // 截图 GestureDetector( onTap: () async { - await logic.captureAndSavePng(); + if (state.talkStatus.value == TalkStatus.duringCall) { + await logic.captureAndSavePng(); + } }, child: Container( width: 50.w, @@ -188,10 +196,12 @@ class _TalkViewPageState extends State // 录制 GestureDetector( onTap: () async { - if (state.isRecording.value) { - await logic.stopRecording(); - } else { - await logic.startRecording(); + if (state.talkStatus.value == TalkStatus.duringCall) { + if (state.isRecordingScreen.value) { + await logic.stopRecording(); + } else { + await logic.startRecording(); + } } }, child: Container( @@ -234,9 +244,15 @@ class _TalkViewPageState extends State Colors.white, longPress: () async { if (state.talkStatus.value == TalkStatus.answeredSuccessfully || - state.talkStatus.value == TalkStatus.duringCall) {} + state.talkStatus.value == TalkStatus.duringCall) { + print('开始录音'); + logic.startProcessingAudio(); + } + }, + longPressUp: () async { + print('停止录音'); + logic.stopProcessingAudio(); }, - longPressUp: () async {}, onClick: () async { if (state.talkStatus.value == TalkStatus.waitingAnswer) { // 接听 diff --git a/lib/talk/startChart/views/talkView/talk_view_state.dart b/lib/talk/startChart/views/talkView/talk_view_state.dart index 896cbfa4..87db2eac 100644 --- a/lib/talk/startChart/views/talkView/talk_view_state.dart +++ b/lib/talk/startChart/views/talkView/talk_view_state.dart @@ -21,7 +21,6 @@ enum NetworkStatus { } class TalkViewState { - int udpSendDataFrameNumber = 0; // 帧序号 // var isSenderAudioData = false.obs;// 是否要发送音频数据 @@ -35,7 +34,6 @@ class TalkViewState { Rx listData = Uint8List(0).obs; //得到的视频流字节数据 RxList listAudioData = [].obs; //得到的音频流字节数据 GlobalKey globalKey = GlobalKey(); - late final VoiceProcessor? voiceProcessor; late Timer oneMinuteTimeTimer = Timer(const Duration(seconds: 1), () {}); // 定时器超过60秒关闭当前界面 @@ -47,16 +45,12 @@ class TalkViewState { late Timer openDoorTimer; late AnimationController animationController; - late Timer autoBackTimer = Timer(const Duration(seconds: 1), () {}); //发送30秒监视后自动返回 late Timer realTimePicTimer = Timer(const Duration(seconds: 1), () {}); //监视命令定时器 RxInt elapsedSeconds = 0.obs; - - - // 星图对讲相关状态 List audioBuffer = [].obs; List videoBuffer = [].obs; @@ -65,6 +59,7 @@ class TalkViewState { // 获取 startChartTalkStatus 的唯一实例 final StartChartTalkStatus startChartTalkStatus = StartChartTalkStatus.instance; + // 通话数据流的单例流数据处理类 final TalkDataRepository talkDataRepository = TalkDataRepository.instance; RxInt lastFrameTimestamp = 0.obs; // 上一帧的时间戳,用来判断网络环境 @@ -73,7 +68,14 @@ class TalkViewState { RxInt alertCount = 0.obs; // 网络状态提示计数器 RxInt maxAlertNumber = 3.obs; // 网络状态提示最大提示次数 RxBool isOpenVoice = true.obs; // 是否打开声音 - RxBool isRecording = true.obs; // 是否录屏中 + RxBool isRecordingScreen = false.obs; // 是否录屏中 + RxBool isRecordingAudio = false.obs; // 是否录音中 + Rx startRecordingAudioTime = DateTime.now().obs; // 开始录音时间 + Rx endRecordingAudioTime= DateTime.now().obs; // 结束录音时间 + RxInt recordingAudioTime= 0.obs; // 录音时间持续时间 RxDouble fps = 0.0.obs; // 添加 FPS 计数 - + late VoiceProcessor? voiceProcessor; // 音频处理器、录音 + final int frameLength = 320; //录音视频帧长度为320 + final int sampleRate = 8000; //录音频采样率为8000 + List> recordingAudioAllFrames = >[]; // 录制音频的所有帧 }