sky/app-starlock
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix:增加录音发送功能

Browse Source
This commit is contained in:
liyi 2024-12-30 11:53:42 +08:00
parent c865db7a9f
commit e850179c92
8 changed files with 270 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lib/talk/startChart/command/message_command.dart
View File

@ -270,10 +270,12 @@ class MessageCommand {
static List<int> talkDataMessage({
required String FromPeerId,
required String ToPeerId,
required TalkData talkData,
int? MessageId,
List<int>? payload,
int? SpTotal,
int? SpIndex,
}) {
final payload = talkData.writeToBuffer();
// final payload = talkData.writeToBuffer();
ScpMessage message = ScpMessage(
ProtocolFlag: ProtocolFlagConstant.scp01,
MessageType: MessageTypeConstant.RealTimeData,
@ -283,7 +285,7 @@ class MessageCommand {
FromPeerId: FromPeerId,
ToPeerId: ToPeerId,
Payload: payload,
PayloadCRC: calculationCrc(payload),
PayloadCRC: calculationCrc(Uint8List.fromList(payload!)),
PayloadLength: payload.length,
PayloadType: PayloadTypeConstant.talkData,
);

4
lib/talk/startChart/handle/impl/udp_go_online_handler.dart
View File

@ -25,7 +25,7 @@ class UdpGoOnlineHandler extends ScpMessageBaseHandle
startChartManage.isOnlineStartChartServer = true;
// 上线成功，停止重发
startChartManage.stopReStartOnlineStartChartServer();
log(text: '星图登录成功');
log(text: '星图登录成功,PeerID:${scpMessage.ToPeerId}');
} else {
// 上线失败，重发
startChartManage.reStartOnlineStartChartServer();
@ -42,7 +42,7 @@ class UdpGoOnlineHandler extends ScpMessageBaseHandle
deserializePayload(
{required int payloadType,
required int messageType,
required List<int> byte,
required List<int> byte,
int? offset,
int? PayloadLength,
int? spTotal,

1
lib/talk/startChart/handle/impl/udp_talk_ping_handler.dart
View File

@ -14,6 +14,7 @@ class UdpTalkPingHandler extends ScpMessageBaseHandle
@override
void handleReq(ScpMessage scpMessage) {
// TODO: 收到通话保持请求
replySuccessMessage(scpMessage);
}
@override

11
lib/talk/startChart/handle/scp_message_base_handle.dart
View File

@ -38,17 +38,17 @@ class ScpMessageBaseHandle {
// 通话请求超时未处理监听定时器管理
final talkeRequestOverTimeTimerManager = OverTimeTimerManager(
timeoutInSeconds: 30,
timeoutInSeconds: 8,
);
// 通话保持超时监听定时器管理
final talkePingOverTimeTimerManager = OverTimeTimerManager(
timeoutInSeconds: 260,
timeoutInSeconds: 5,
);
// 通话数据超时定时器
final talkDataOverTimeTimerManager = OverTimeTimerManager(
timeoutInSeconds: 260,
timeoutInSeconds: 30,
);
// 回复成功消息
@ -64,8 +64,9 @@ class ScpMessageBaseHandle {
if (genericResp == null) return false;
final code = genericResp.code;
final message = genericResp.message;
return code == UdpConstant.genericRespSuccessCode &&
message == UdpConstant.genericRespSuccessMsg;
return (code == UdpConstant.genericRespSuccessCode || code == null) &&
message.toLowerCase() ==
UdpConstant.genericRespSuccessMsg.toLowerCase();
}
void log({required String text}) {

66
lib/talk/startChart/start_chart_manage.dart
View File

@ -92,7 +92,7 @@ class StartChartManage {
);
// 默认通话数据
TalkData defaultTalkData = TalkData();
TalkData _defaultTalkData = TalkData();
String relayPeerId = ''; // 中继peerId
@ -252,14 +252,44 @@ class StartChartManage {
// 发送对讲数据
Future<void> sendTalkDataMessage({required TalkData talkData}) async {
// 组装上线消息
final message = MessageCommand.talkDataMessage(
FromPeerId: FromPeerId,
ToPeerId: ToPeerId,
talkData: talkData,
MessageId: MessageCommand.getNextMessageId(ToPeerId, increment: true),
);
await _sendMessage(message: message);
final List<int> payload = talkData.content;
// 计算需要分多少个包发送
final int totalPackets = (payload.length / _maxPayloadSize).ceil();
// 循环遍历
for (int i = 0; i < totalPackets; i++) {
int start = i * _maxPayloadSize;
int end = (i + 1) * _maxPayloadSize;
if (end > payload.length) {
end = payload.length;
}
// 取到分包数据
List<int> packet = payload.sublist(start, end);
// 分包数据不递增messageID
final messageId =
MessageCommand.getNextMessageId(ToPeerId, increment: false);
// 组装分包数据
final message = MessageCommand.talkDataMessage(
ToPeerId: ToPeerId,
FromPeerId: FromPeerId,
payload: packet,
SpTotal: totalPackets,
SpIndex: i + 1,
MessageId: messageId,
);
// 发送消息
await _sendMessage(message: message);
}
// 分包发送完了递增一下id
MessageCommand.getNextMessageId(ToPeerId);
//
// final message = MessageCommand.talkDataMessage(
// FromPeerId: FromPeerId,
// ToPeerId: ToPeerId,
// talkData: talkData,
// MessageId: MessageCommand.getNextMessageId(ToPeerId, increment: true),
// );
// await _sendMessage(message: message);
}
// 发送心跳包消息
@ -350,7 +380,7 @@ class StartChartManage {
await _sendMessage(message: message);
// 设置状态为等待接听
talkStatus.setWaitingAnswer();
_log(text: '发送同意接听消息');
// _log(text: '发送同意接听消息');
}
// 发送拒绝接听消息
@ -381,6 +411,7 @@ class StartChartManage {
MessageId: MessageCommand.getNextMessageId(ToPeerId, increment: true),
);
await _sendMessage(message: message);
// _log(text: '发送预期数据：${talkExpect}');
}
// 回复成功消息
@ -420,6 +451,7 @@ class StartChartManage {
MessageId: MessageCommand.getNextMessageId(ToPeerId, increment: true),
);
await _sendMessage(message: message);
_log(text: '发送通话保持');
}
// 发送通话中挂断消息
@ -892,13 +924,13 @@ class StartChartManage {
current++;
List<int> frameData = byteData.sublist(start, end);
if (frameData.length == 0) timer.cancel();
defaultTalkData = TalkData(
_defaultTalkData = TalkData(
content: frameData,
contentType: TalkData_ContentTypeE.H264,
);
start = end;
// 发送童话数据
sendTalkDataMessage(talkData: defaultTalkData);
// 发送通话数据
sendTalkDataMessage(talkData: _defaultTalkData);
},
);
}
@ -991,7 +1023,15 @@ class StartChartManage {
stopHeartbeat();
stopReStartOnlineStartChartServer();
stopTalkDataTimer();
_resetData();
// await Storage.removerRelayInfo();
// await Storage.removerStarChartRegisterNodeInfo();
}
void _resetData() {
_defaultTalkExpect = TalkExpectReq(
videoType: [VideoTypeE.IMAGE],
audioType: [AudioTypeE.G711],
);
}
}

155
lib/talk/startChart/views/talkView/talk_view_logic.dart
View File

@ -7,6 +7,8 @@ import 'package:flutter/services.dart';
import 'package:flutter_easyloading/flutter_easyloading.dart';
import 'package:flutter_pcm_sound/flutter_pcm_sound.dart';
import 'package:flutter_screen_recording/flutter_screen_recording.dart';
import 'package:flutter_voice_processor/flutter_voice_processor.dart';
import 'package:gallery_saver/gallery_saver.dart';
import 'package:get/get.dart';
@ -31,14 +33,12 @@ class TalkViewLogic extends BaseGetXController {
Timer? _syncTimer;
int _startTime = 0;
final int bufferSize = 20; // 缓冲区大小（以帧为单位）
final List<int> frameTimestamps = [];
final List<int> frameTimestamps = []; // 帧时间戳用于计算 FPS
int frameIntervalMs = 45; // 初始帧间隔设置为45毫秒（约22FPS）
int minFrameIntervalMs = 30; // 最小帧间隔（约33 FPS）
int maxFrameIntervalMs = 100; // 最大帧间隔（约10 FPS）
/// 收到Talk发送的状态
StreamSubscription? _getTalkStatusRefreshUIEvent;
/// 初始化音频播放器
void _initFlutterPcmSound() {
const int sampleRate = 44100;
FlutterPcmSound.setLogLevel(LogLevel.verbose);
@ -129,6 +129,7 @@ class TalkViewLogic extends BaseGetXController {
});
}
/// 播放音频数据
void _playAudioData(TalkData talkData) {
// 将 PCM 数据转换为 PcmArrayInt16
final PcmArrayInt16 fromList = PcmArrayInt16.fromList(talkData.content);
@ -139,6 +140,7 @@ class TalkViewLogic extends BaseGetXController {
}
}
/// 播放视频数据
void _playVideoData(TalkData talkData) {
state.listData.value = Uint8List.fromList(talkData.content);
}
@ -263,7 +265,9 @@ class TalkViewLogic extends BaseGetXController {
/// 提示网络状态
void showNetworkStatus(String message) {
if (state.alertCount.value < 3 && !EasyLoading.isShow) {
// 如果提示次数未达到最大值且 EasyLoading 未显示，则显示提示
if (state.alertCount.value < state.maxAlertNumber.value &&
!EasyLoading.isShow) {
showToast(message);
state.alertCount++;
}
@ -279,6 +283,7 @@ class TalkViewLogic extends BaseGetXController {
/// 开门
udpOpenDoorAction(List<int> list) async {}
/// 获取权限状态
Future<bool> getPermissionStatus() async {
final Permission permission = Permission.microphone;
//granted 通过，denied 被拒绝，permanentlyDenied 拒绝且不在提示
@ -312,17 +317,22 @@ class TalkViewLogic extends BaseGetXController {
void onInit() {
super.onInit();
// 监听音视频数据流
// 启动监听音视频数据流
_startListenTalkData();
// 监听对讲状态
// 启动监听对讲状态
_startListenTalkStatus();
// 在没有监听成功之前赋值一遍状态
// *** 由于页面会在状态变化之后才会初始化，导致识别不到最新的状态，在这里手动赋值 ***
state.talkStatus.value = state.startChartTalkStatus.status;
// 初始化音频播放器
_initFlutterPcmSound();
// 启动播放定时器
_startPlayback();
// 初始化录音控制器
_initAudioRecorder();
}
@override
@ -404,7 +414,7 @@ class TalkViewLogic extends BaseGetXController {
bool started =
await FlutterScreenRecording.startRecordScreenAndAudio("Recording");
if (started) {
state.isRecording.value = true;
state.isRecordingScreen.value = true;
}
}
@ -412,13 +422,136 @@ class TalkViewLogic extends BaseGetXController {
Future<void> stopRecording() async {
String path = await FlutterScreenRecording.stopRecordScreen;
if (path != null) {
state.isRecording.value = false;
state.isRecordingScreen.value = false;
// 保存录制的视频到相册
await GallerySaver.saveVideo(path).then((bool? success) {});
// await GallerySaver.saveVideo(path).then((bool? success) {});
// 将截图保存到相册
await ImageGallerySaver.saveFile(path);
showToast('录屏已保存到相册'.tr);
} else {
state.isRecording.value = false;
state.isRecordingScreen.value = false;
print("Recording failed");
}
}
/// 初始化音频录制器
void _initAudioRecorder() {
state.voiceProcessor = VoiceProcessor.instance;
}
//开始录音
Future<void> startProcessingAudio() async {
// 增加录音帧监听器和错误监听器
state.voiceProcessor?.addFrameListener(_onFrame);
state.voiceProcessor?.addErrorListener(_onError);
try {
if (await state.voiceProcessor?.hasRecordAudioPermission() ?? false) {
await state.voiceProcessor?.start(state.frameLength, state.sampleRate);
final bool? isRecording = await state.voiceProcessor?.isRecording();
state.isRecordingAudio.value = isRecording!;
state.startRecordingAudioTime.value = DateTime.now();
} else {
// state.errorMessage.value = 'Recording permission not granted';
}
} on PlatformException catch (ex) {
// state.errorMessage.value = 'Failed to start recorder: $ex';
}
}
/// 停止录音
Future<void> stopProcessingAudio() async {
try {
await state.voiceProcessor?.stop();
state.voiceProcessor?.removeFrameListener(_onFrame);
state.udpSendDataFrameNumber = 0;
// 记录结束时间
state.endRecordingAudioTime.value = DateTime.now();
// 计算录音的持续时间
final duration = state.endRecordingAudioTime.value!
.difference(state.startRecordingAudioTime.value!);
state.recordingAudioTime.value = duration.inSeconds;
} on PlatformException catch (ex) {
// state.errorMessage.value = 'Failed to stop recorder: $ex';
} finally {
final bool? isRecording = await state.voiceProcessor?.isRecording();
state.isRecordingAudio.value = isRecording!;
}
}
Future<void> _onFrame(List<int> frame) async {
state.recordingAudioAllFrames.add(frame); // 将帧添加到状态变量中
// final List<int> concatenatedFrames =
// _concatenateFrames(state.recordingAudioAllFrames); // 连接所有帧
final List<int> pcmBytes = _listLinearToULaw(frame);
// 发送音频数据
StartChartManage().sendTalkDataMessage(
talkData: TalkData(
content: pcmBytes,
contentType: TalkData_ContentTypeE.G711,
durationMs: DateTime.now().millisecondsSinceEpoch -
state.startRecordingAudioTime.value.millisecondsSinceEpoch,
),
);
}
void _onError(VoiceProcessorException error) {
// state.errorMessage.value = error.message!;
AppLog.log(error.message!);
}
// 拿到的音频转化成pcm
List<int> _listLinearToULaw(List<int> pcmList) {
final List<int> uLawList = [];
for (int pcmVal in pcmList) {
final int uLawVal = _linearToULaw(pcmVal);
uLawList.add(uLawVal);
}
return uLawList;
}
// 拿到的音频转化成pcm
int _linearToULaw(int pcmVal) {
int mask;
int seg;
int uval;
if (pcmVal < 0) {
pcmVal = 0x84 - pcmVal;
mask = 0x7F;
} else {
pcmVal += 0x84;
mask = 0xFF;
}
seg = search(pcmVal);
if (seg >= 8) {
return 0x7F ^ mask;
} else {
uval = seg << 4;
uval |= (pcmVal >> (seg + 3)) & 0xF;
return uval ^ mask;
}
}
int search(int val) {
final List<int> table = [
0xFF,
0x1FF,
0x3FF,
0x7FF,
0xFFF,
0x1FFF,
0x3FFF,
0x7FFF
];
const int size = 8;
for (int i = 0; i < size; i++) {
if (val <= table[i]) {
return i;
}
}
return size;
}
}

82
lib/talk/startChart/views/talkView/talk_view_page.dart
View File

@ -5,17 +5,14 @@ import 'dart:io';
import 'package:flutter/material.dart';
import 'package:flutter/rendering.dart';
import 'package:flutter/services.dart';
import 'package:flutter_screen_recording/flutter_screen_recording.dart';
import 'package:flutter_screenutil/flutter_screenutil.dart';
import 'package:gallery_saver/gallery_saver.dart';
import 'package:get/get.dart';
import 'package:image_gallery_saver/image_gallery_saver.dart';
import 'package:path_provider/path_provider.dart';
import 'package:star_lock/app_settings/app_settings.dart';
import 'package:star_lock/main/lockDetail/realTimePicture/realTimePicture_state.dart';
import 'package:star_lock/talk/call/callTalk.dart';
import 'package:star_lock/talk/startChart/constant/talk_status.dart';
import 'package:star_lock/talk/startChart/start_chart_talk_status.dart';
import 'package:star_lock/talk/startChart/views/talkView/talk_view_logic.dart';
import 'package:star_lock/talk/startChart/views/talkView/talk_view_state.dart';
@ -75,24 +72,30 @@ class _TalkViewPageState extends State<TalkViewPage>
height: ScreenUtil().screenHeight,
fit: BoxFit.cover,
)
: PopScope(
canPop: false,
child: RepaintBoundary(
key: state.globalKey,
child: Image.memory(
state.listData.value,
gaplessPlayback: true,
width: 1.sw,
height: 1.sh,
fit: BoxFit.cover,
filterQuality: FilterQuality.high,
errorBuilder: (
BuildContext context,
Object error,
StackTrace? stackTrace,
) {
return Container(color: Colors.transparent);
},
: Container(
decoration: state.isRecordingScreen.value
? BoxDecoration(
border: Border.all(color: Colors.red, width: 1))
: BoxDecoration(),
child: PopScope(
canPop: false,
child: RepaintBoundary(
key: state.globalKey,
child: Image.memory(
state.listData.value,
gaplessPlayback: true,
width: 1.sw,
height: 1.sh,
fit: BoxFit.cover,
filterQuality: FilterQuality.high,
errorBuilder: (
BuildContext context,
Object error,
StackTrace? stackTrace,
) {
return Container(color: Colors.transparent);
},
),
),
),
),
@ -151,7 +154,10 @@ class _TalkViewPageState extends State<TalkViewPage>
// 打开关闭声音
GestureDetector(
onTap: () {
logic.updateTalkExpect();
if (state.talkStatus.value == TalkStatus.duringCall) {
// 打开关闭声音
logic.updateTalkExpect();
}
},
child: Container(
width: 50.w,
@ -171,7 +177,9 @@ class _TalkViewPageState extends State<TalkViewPage>
// 截图
GestureDetector(
onTap: () async {
await logic.captureAndSavePng();
if (state.talkStatus.value == TalkStatus.duringCall) {
await logic.captureAndSavePng();
}
},
child: Container(
width: 50.w,
@ -188,10 +196,12 @@ class _TalkViewPageState extends State<TalkViewPage>
// 录制
GestureDetector(
onTap: () async {
if (state.isRecording.value) {
await logic.stopRecording();
} else {
await logic.startRecording();
if (state.talkStatus.value == TalkStatus.duringCall) {
if (state.isRecordingScreen.value) {
await logic.stopRecording();
} else {
await logic.startRecording();
}
}
},
child: Container(
@ -234,9 +244,15 @@ class _TalkViewPageState extends State<TalkViewPage>
Colors.white,
longPress: () async {
if (state.talkStatus.value == TalkStatus.answeredSuccessfully ||
state.talkStatus.value == TalkStatus.duringCall) {}
state.talkStatus.value == TalkStatus.duringCall) {
print('开始录音');
logic.startProcessingAudio();
}
},
longPressUp: () async {
print('停止录音');
logic.stopProcessingAudio();
},
longPressUp: () async {},
onClick: () async {
if (state.talkStatus.value == TalkStatus.waitingAnswer) {
// 接听

18
lib/talk/startChart/views/talkView/talk_view_state.dart
View File

@ -21,7 +21,6 @@ enum NetworkStatus {
}
class TalkViewState {
int udpSendDataFrameNumber = 0; // 帧序号
// var isSenderAudioData = false.obs;// 是否要发送音频数据
@ -35,7 +34,6 @@ class TalkViewState {
Rx<Uint8List> listData = Uint8List(0).obs; //得到的视频流字节数据
RxList<int> listAudioData = <int>[].obs; //得到的音频流字节数据
GlobalKey globalKey = GlobalKey();
late final VoiceProcessor? voiceProcessor;
late Timer oneMinuteTimeTimer =
Timer(const Duration(seconds: 1), () {}); // 定时器超过60秒关闭当前界面
@ -47,16 +45,12 @@ class TalkViewState {
late Timer openDoorTimer;
late AnimationController animationController;
late Timer autoBackTimer =
Timer(const Duration(seconds: 1), () {}); //发送30秒监视后自动返回
late Timer realTimePicTimer =
Timer(const Duration(seconds: 1), () {}); //监视命令定时器
RxInt elapsedSeconds = 0.obs;
// 星图对讲相关状态
List<TalkData> audioBuffer = <TalkData>[].obs;
List<TalkData> videoBuffer = <TalkData>[].obs;
@ -65,6 +59,7 @@ class TalkViewState {
// 获取 startChartTalkStatus 的唯一实例
final StartChartTalkStatus startChartTalkStatus =
StartChartTalkStatus.instance;
// 通话数据流的单例流数据处理类
final TalkDataRepository talkDataRepository = TalkDataRepository.instance;
RxInt lastFrameTimestamp = 0.obs; // 上一帧的时间戳,用来判断网络环境
@ -73,7 +68,14 @@ class TalkViewState {
RxInt alertCount = 0.obs; // 网络状态提示计数器
RxInt maxAlertNumber = 3.obs; // 网络状态提示最大提示次数
RxBool isOpenVoice = true.obs; // 是否打开声音
RxBool isRecording = true.obs; // 是否录屏中
RxBool isRecordingScreen = false.obs; // 是否录屏中
RxBool isRecordingAudio = false.obs; // 是否录音中
Rx<DateTime> startRecordingAudioTime = DateTime.now().obs; // 开始录音时间
Rx<DateTime> endRecordingAudioTime= DateTime.now().obs; // 结束录音时间
RxInt recordingAudioTime= 0.obs; // 录音时间持续时间
RxDouble fps = 0.0.obs; // 添加 FPS 计数
late VoiceProcessor? voiceProcessor; // 音频处理器、录音
final int frameLength = 320; //录音视频帧长度为320
final int sampleRate = 8000; //录音频采样率为8000
List<List<int>> recordingAudioAllFrames = <List<int>>[]; // 录制音频的所有帧
}