From 51ca6e1f23593e006b99058b245f117261d2eefb Mon Sep 17 00:00:00 2001 From: liyi Date: Wed, 18 Jun 2025 15:12:51 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8D=E5=AF=B9=E8=AE=B2?= =?UTF-8?q?=E7=9A=84=E9=AB=98=E6=B8=85=E3=80=81=E6=A0=87=E6=B8=85=E5=88=87?= =?UTF-8?q?=E6=8D=A2=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../native/talk_view_native_decode_logic.dart | 417 +++++++++++------- 1 file changed, 260 insertions(+), 157 deletions(-) diff --git a/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart b/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart index 9ddf4a57..53dc4837 100644 --- a/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart +++ b/lib/talk/starChart/views/native/talk_view_native_decode_logic.dart @@ -25,6 +25,7 @@ import 'package:star_lock/network/api_repository.dart'; import 'package:star_lock/talk/call/callTalk.dart'; import 'package:star_lock/talk/call/g711.dart'; import 'package:star_lock/talk/starChart/constant/talk_status.dart'; +import 'package:star_lock/talk/starChart/entity/scp_message.dart'; import 'package:star_lock/talk/starChart/handle/other/packet_loss_statistics.dart'; import 'package:star_lock/talk/starChart/handle/other/talk_data_model.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; @@ -87,15 +88,17 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { // 新增：等待新I帧状态 bool _waitingForIFrame = false; + int? lastDecodedIFrameSeq; + // 初始化视频解码器 Future _initVideoDecoder() async { try { state.isLoading.value = true; // 创建解码器配置 final config = VideoDecoderConfig( - width: 864, + width: StartChartManage().videoWidth, // 实际视频宽度 - height: 480, + height: StartChartManage().videoHeight, codecType: 'h264', ); // 初始化解码器并获取textureId @@ -157,6 +160,7 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { int pts, int frameSeq, int frameSeqI, + ScpMessage scpMessage, ) { // 检测frameSeq回绕，且为I帧 if (!_pendingStreamReset && @@ -212,6 +216,7 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { 'frameSeq': frameSeq, 'frameSeqI': frameSeqI, 'pts': pts, + 'scpMessage': scpMessage, }; // 如果缓冲区超出最大大小，优先丢弃P/B帧 @@ -257,14 +262,25 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { return; } - // 设置正在处理标志 - state.isProcessingFrame = true; + // 优先查找I帧，按frameSeq最小的I帧消费 + final iFrames = state.h264FrameBuffer + .where((f) => f['frameType'] == TalkDataH264Frame_FrameTypeE.I) + .toList(); + iFrames + .sort((a, b) => (a['frameSeq'] as int).compareTo(b['frameSeq'] as int)); - try { - // 取出最早的帧 - final Map? frameMap = state.h264FrameBuffer.isNotEmpty - ? state.h264FrameBuffer.removeAt(0) - : null; + if (iFrames.isNotEmpty) { + // 有I帧，消费最小的I帧，并记录其frameSeq + final minIFrame = iFrames.first; + final minIFrameSeq = minIFrame['frameSeq']; + final targetIndex = state.h264FrameBuffer.indexWhere( + (f) => + f['frameType'] == TalkDataH264Frame_FrameTypeE.I && + f['frameSeq'] == minIFrameSeq, + ); + state.isProcessingFrame = true; + final Map? frameMap = + state.h264FrameBuffer.removeAt(targetIndex); if (frameMap == null) { state.isProcessingFrame = false; return; @@ -274,6 +290,7 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { final int? frameSeq = frameMap['frameSeq']; final int? frameSeqI = frameMap['frameSeqI']; final int? pts = frameMap['pts']; + final ScpMessage? scpMessage = frameMap['scpMessage']; if (frameData == null || frameType == null || frameSeq == null || @@ -282,25 +299,82 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { state.isProcessingFrame = false; return; } - // 解码器未初始化或textureId为null时跳过 if (state.textureId.value == null) { state.isProcessingFrame = false; return; } + lastDecodedIFrameSeq = minIFrameSeq; + // AppLog.log('送入解码器的P帧数据frameSeq:${frameSeq},frameSeqI:${frameSeqI},' + // 'frameType:${frameType},messageId:${scpMessage!.MessageId}'); await VideoDecodePlugin.sendFrame( frameData: frameData, - frameType: frameType == TalkDataH264Frame_FrameTypeE.I ? 0 : 1, + frameType: 0, frameSeq: frameSeq, timestamp: pts, splitNalFromIFrame: true, refIFrameSeq: frameSeqI, ); - } catch (e) { - AppLog.log('处理缓冲帧失败: $e'); - } finally { - // 重置处理标志 state.isProcessingFrame = false; + return; } + + // 没有I帧时，只消费refIFrameSeq等于lastDecodedIFrameSeq的P帧 + if (lastDecodedIFrameSeq != null) { + final validPFrames = state.h264FrameBuffer + .where((f) => + f['frameType'] == TalkDataH264Frame_FrameTypeE.P && + f['frameSeqI'] == lastDecodedIFrameSeq) + .toList(); + if (validPFrames.isNotEmpty) { + validPFrames.sort( + (a, b) => (a['frameSeq'] as int).compareTo(b['frameSeq'] as int)); + final minPFrame = validPFrames.first; + final targetIndex = state.h264FrameBuffer.indexWhere( + (f) => + f['frameType'] == TalkDataH264Frame_FrameTypeE.P && + f['frameSeq'] == minPFrame['frameSeq'] && + f['frameSeqI'] == lastDecodedIFrameSeq, + ); + state.isProcessingFrame = true; + final Map? frameMap = + state.h264FrameBuffer.removeAt(targetIndex); + if (frameMap == null) { + state.isProcessingFrame = false; + return; + } + final List? frameData = frameMap['frameData']; + final TalkDataH264Frame_FrameTypeE? frameType = frameMap['frameType']; + final int? frameSeq = frameMap['frameSeq']; + final int? frameSeqI = frameMap['frameSeqI']; + final int? pts = frameMap['pts']; + final ScpMessage? scpMessage = frameMap['scpMessage']; + if (frameData == null || + frameType == null || + frameSeq == null || + frameSeqI == null || + pts == null) { + state.isProcessingFrame = false; + return; + } + if (state.textureId.value == null) { + state.isProcessingFrame = false; + return; + } + // AppLog.log('送入解码器的P帧数据frameSeq:${frameSeq},frameSeqI:${frameSeqI},' + // 'frameType:${frameType},messageId:${scpMessage!.MessageId}'); + await VideoDecodePlugin.sendFrame( + frameData: frameData, + frameType: 1, + frameSeq: frameSeq, + timestamp: pts, + splitNalFromIFrame: true, + refIFrameSeq: frameSeqI, + ); + state.isProcessingFrame = false; + return; + } + } + // 其他情况不消费，等待I帧到来 } /// 停止帧处理定时器 @@ -331,6 +405,7 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { .listen((TalkDataModel talkDataModel) async { final talkData = talkDataModel.talkData; final talkDataH264Frame = talkDataModel.talkDataH264Frame; + final scpMessage = talkDataModel.scpMessage; final contentType = talkData!.contentType; // 判断数据类型，进行分发处理 @@ -345,7 +420,7 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { break; case TalkData_ContentTypeE.H264: // 处理H264帧 - if (state.textureId.value != null) { + if (state.textureId.value != null || true) { if (talkDataH264Frame != null) { _addFrameToBuffer( talkData.content, @@ -353,6 +428,7 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { talkData.durationMs, talkDataH264Frame.frameSeq, talkDataH264Frame.frameSeqI, + scpMessage!, ); } } else { @@ -585,7 +661,8 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { TalkExpectReq talkExpectReq = TalkExpectReq(); state.isOpenVoice.value = !state.isOpenVoice.value; // 根据当前清晰度动态设置videoType - VideoTypeE currentVideoType = qualityToVideoType[state.currentQuality.value] ?? VideoTypeE.H264; + VideoTypeE currentVideoType = + qualityToVideoType[state.currentQuality.value] ?? VideoTypeE.H264; if (!state.isOpenVoice.value) { talkExpectReq = TalkExpectReq( videoType: [currentVideoType], @@ -1104,144 +1181,144 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { } // 新增：I帧处理方法 - void _handleIFrameWithSpsPpsAndIdr( - List frameData, int durationMs, int frameSeq, int frameSeqI) { - // 清空缓冲区，丢弃I帧前所有未处理帧（只保留SPS/PPS/I帧） - state.h264FrameBuffer.clear(); - _extractAndBufferSpsPpsForBuffer( - frameData, durationMs, frameSeq, frameSeqI); - // 只要缓存有SPS/PPS就先写入，再写I帧本体（只写IDR） - if (spsCache == null || ppsCache == null) { - // 没有SPS/PPS缓存，丢弃本次I帧 - return; - } - // 先写入SPS/PPS - _addFrameToBuffer(spsCache!, TalkDataH264Frame_FrameTypeE.I, durationMs, - frameSeq, frameSeqI); - _addFrameToBuffer(ppsCache!, TalkDataH264Frame_FrameTypeE.I, durationMs, - frameSeq, frameSeqI); - // 分割I帧包，只写入IDR（type 5） - List> nalus = []; - int i = 0; - List data = frameData; - while (i < data.length - 3) { - int start = -1; - int next = -1; - if (data[i] == 0x00 && data[i + 1] == 0x00) { - if (data[i + 2] == 0x01) { - start = i; - i += 3; - } else if (i + 3 < data.length && - data[i + 2] == 0x00 && - data[i + 3] == 0x01) { - start = i; - i += 4; - } else { - i++; - continue; - } - next = i; - while (next < data.length - 3) { - if (data[next] == 0x00 && - data[next + 1] == 0x00 && - ((data[next + 2] == 0x01) || - (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { - break; - } - next++; - } - nalus.add(data.sublist(start, next)); - i = next; - } else { - i++; - } - } - int nalusTotalLen = - nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; - if (nalus.isEmpty && data.isNotEmpty) { - nalus.add(data); - } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { - nalus.add(data.sublist(nalusTotalLen)); - } - for (final nalu in nalus) { - int offset = 0; - if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { - if (nalu[2] == 0x01) - offset = 3; - else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; - } - if (nalu.length > offset) { - int naluType = nalu[offset] & 0x1F; - if (naluType == 5) { - _addFrameToBuffer(nalu, TalkDataH264Frame_FrameTypeE.I, durationMs, - frameSeq, frameSeqI); - } - } - } - } + // void _handleIFrameWithSpsPpsAndIdr( + // List frameData, int durationMs, int frameSeq, int frameSeqI) { + // // 清空缓冲区，丢弃I帧前所有未处理帧（只保留SPS/PPS/I帧） + // state.h264FrameBuffer.clear(); + // _extractAndBufferSpsPpsForBuffer( + // frameData, durationMs, frameSeq, frameSeqI); + // // 只要缓存有SPS/PPS就先写入，再写I帧本体（只写IDR） + // if (spsCache == null || ppsCache == null) { + // // 没有SPS/PPS缓存，丢弃本次I帧 + // return; + // } + // // 先写入SPS/PPS + // _addFrameToBuffer(spsCache!, TalkDataH264Frame_FrameTypeE.I, durationMs, + // frameSeq, frameSeqI); + // _addFrameToBuffer(ppsCache!, TalkDataH264Frame_FrameTypeE.I, durationMs, + // frameSeq, frameSeqI); + // // 分割I帧包，只写入IDR（type 5） + // List> nalus = []; + // int i = 0; + // List data = frameData; + // while (i < data.length - 3) { + // int start = -1; + // int next = -1; + // if (data[i] == 0x00 && data[i + 1] == 0x00) { + // if (data[i + 2] == 0x01) { + // start = i; + // i += 3; + // } else if (i + 3 < data.length && + // data[i + 2] == 0x00 && + // data[i + 3] == 0x01) { + // start = i; + // i += 4; + // } else { + // i++; + // continue; + // } + // next = i; + // while (next < data.length - 3) { + // if (data[next] == 0x00 && + // data[next + 1] == 0x00 && + // ((data[next + 2] == 0x01) || + // (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { + // break; + // } + // next++; + // } + // nalus.add(data.sublist(start, next)); + // i = next; + // } else { + // i++; + // } + // } + // int nalusTotalLen = + // nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; + // if (nalus.isEmpty && data.isNotEmpty) { + // nalus.add(data); + // } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { + // nalus.add(data.sublist(nalusTotalLen)); + // } + // for (final nalu in nalus) { + // int offset = 0; + // if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { + // if (nalu[2] == 0x01) + // offset = 3; + // else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; + // } + // if (nalu.length > offset) { + // int naluType = nalu[offset] & 0x1F; + // if (naluType == 5) { + // _addFrameToBuffer(nalu, TalkDataH264Frame_FrameTypeE.I, durationMs, + // frameSeq, frameSeqI); + // } + // } + // } + // } // 新增：P帧处理方法 - void _handlePFrame( - List frameData, int durationMs, int frameSeq, int frameSeqI) { - // 只写入P帧（type 1） - List> nalus = []; - int i = 0; - List data = frameData; - while (i < data.length - 3) { - int start = -1; - int next = -1; - if (data[i] == 0x00 && data[i + 1] == 0x00) { - if (data[i + 2] == 0x01) { - start = i; - i += 3; - } else if (i + 3 < data.length && - data[i + 2] == 0x00 && - data[i + 3] == 0x01) { - start = i; - i += 4; - } else { - i++; - continue; - } - next = i; - while (next < data.length - 3) { - if (data[next] == 0x00 && - data[next + 1] == 0x00 && - ((data[next + 2] == 0x01) || - (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { - break; - } - next++; - } - nalus.add(data.sublist(start, next)); - i = next; - } else { - i++; - } - } - int nalusTotalLen = - nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; - if (nalus.isEmpty && data.isNotEmpty) { - nalus.add(data); - } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { - nalus.add(data.sublist(nalusTotalLen)); - } - for (final nalu in nalus) { - int offset = 0; - if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { - if (nalu[2] == 0x01) - offset = 3; - else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; - } - if (nalu.length > offset) { - int naluType = nalu[offset] & 0x1F; - if (naluType == 1) { - _addFrameToBuffer(nalu, TalkDataH264Frame_FrameTypeE.P, durationMs, - frameSeq, frameSeqI); - } - } - } - } + // void _handlePFrame( + // List frameData, int durationMs, int frameSeq, int frameSeqI) { + // // 只写入P帧（type 1） + // List> nalus = []; + // int i = 0; + // List data = frameData; + // while (i < data.length - 3) { + // int start = -1; + // int next = -1; + // if (data[i] == 0x00 && data[i + 1] == 0x00) { + // if (data[i + 2] == 0x01) { + // start = i; + // i += 3; + // } else if (i + 3 < data.length && + // data[i + 2] == 0x00 && + // data[i + 3] == 0x01) { + // start = i; + // i += 4; + // } else { + // i++; + // continue; + // } + // next = i; + // while (next < data.length - 3) { + // if (data[next] == 0x00 && + // data[next + 1] == 0x00 && + // ((data[next + 2] == 0x01) || + // (data[next + 2] == 0x00 && data[next + 3] == 0x01))) { + // break; + // } + // next++; + // } + // nalus.add(data.sublist(start, next)); + // i = next; + // } else { + // i++; + // } + // } + // int nalusTotalLen = + // nalus.isNotEmpty ? nalus.fold(0, (p, n) => p + n.length) : 0; + // if (nalus.isEmpty && data.isNotEmpty) { + // nalus.add(data); + // } else if (nalus.isNotEmpty && nalusTotalLen < data.length) { + // nalus.add(data.sublist(nalusTotalLen)); + // } + // for (final nalu in nalus) { + // int offset = 0; + // if (nalu.length >= 4 && nalu[0] == 0x00 && nalu[1] == 0x00) { + // if (nalu[2] == 0x01) + // offset = 3; + // else if (nalu[2] == 0x00 && nalu[3] == 0x01) offset = 4; + // } + // if (nalu.length > offset) { + // int naluType = nalu[offset] & 0x1F; + // if (naluType == 1) { + // _addFrameToBuffer(nalu, TalkDataH264Frame_FrameTypeE.P, durationMs, + // frameSeq, frameSeqI); + // } + // } + // } + // } // 切换清晰度的方法，后续补充具体实现 void onQualityChanged(String quality) async { @@ -1293,30 +1370,56 @@ class TalkViewNativeDecodeLogic extends BaseGetXController { // 新增：重置解码器方法 Future _resetDecoderForNewStream(int width, int height) async { try { + // 先停止帧处理定时器 + _stopFrameProcessTimer(); + + // 释放旧解码器 if (state.textureId.value != null) { await VideoDecodePlugin.releaseDecoder(); - Future.microtask(() => state.textureId.value = null); + state.textureId.value = null; } + + // 等待一小段时间确保资源释放完成 + await Future.delayed(Duration(milliseconds: 100)); + + // 创建新的解码器配置 final config = VideoDecoderConfig( width: width, height: height, codecType: 'h264', ); + + // 初始化新解码器 final textureId = await VideoDecodePlugin.initDecoder(config); if (textureId != null) { - Future.microtask(() => state.textureId.value = textureId); + state.textureId.value = textureId; AppLog.log('frameSeq回绕后解码器初始化成功：textureId=$textureId'); + + // 设置帧渲染监听 VideoDecodePlugin.setOnFrameRenderedListener((textureId) { AppLog.log('已经开始渲染======='); // 只有真正渲染出首帧时才关闭loading - Future.microtask(() => state.isLoading.value = false); + state.isLoading.value = false; }); + + // 重新启动帧处理定时器 + _startFrameProcessTimer(); + + // 重置相关状态 + _decodedIFrames.clear(); + state.h264FrameBuffer.clear(); + state.isProcessingFrame = false; + hasSps = false; + hasPps = false; + spsCache = null; + ppsCache = null; } else { AppLog.log('frameSeq回绕后解码器初始化失败'); + state.isLoading.value = false; } - _startFrameProcessTimer(); } catch (e) { AppLog.log('frameSeq回绕时解码器初始化错误: $e'); + state.isLoading.value = false; } } }