diff --git a/README.md b/README.md index 480f9ba1..45dfc905 100755 --- a/README.md +++ b/README.md @@ -171,3 +171,14 @@ java -jar android/bundletool.jar build-apks --bundle=build/app/outputs/bundle/sk ```bash java -jar android/bundletool.jar install-apks --apks=build/app/outputs/bundle/skyRelease/app-sky-release.aab.apks ``` + +## Jpush相关 + +极光推送，目前app这边只依赖极光的透传能力，推送能力通过截取极光拿到的各个厂商的推送token，然后将推送token上报到自己业务服务器直接调用各个厂商推送通道进行消息推送，所以对极光的flutter sdk进行了私有化定制改造，改造点如下： + +* Android，iOS平台原生代码中截取jpush获取到的厂商推送token，将token回传到flutter业务应用层 + * Android通过Jpush统一集成的各个厂商推送sdk，统一获取到token + * iOS通过原生token回调接口获取到token +* flutter端，将获取到的厂商token，厂商标识上报到业务服务器 + +定制jpush_flutter：http://code-internal.star-lock.cn/StarlockTeam/jpush_flutter \ No newline at end of file diff --git a/assets/html/h264.html b/assets/html/h264.html index 191f47a5..97143565 100644 --- a/assets/html/h264.html +++ b/assets/html/h264.html @@ -7,2654 +7,36 @@ play + + - - diff --git a/assets/html/jmuxer.min.js b/assets/html/jmuxer.min.js new file mode 100644 index 00000000..6e31d497 --- /dev/null +++ b/assets/html/jmuxer.min.js @@ -0,0 +1 @@ +!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t(require("stream")):"function"==typeof define&&define.amd?define(["stream"],t):(e="undefined"!=typeof globalThis?globalThis:e||self).JMuxer=t(e.stream)}(this,(function(e){"use strict";function t(e){return t="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e},t(e)}function n(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function r(e,t){for(var n=0;ne.length)&&(t=e.length);for(var n=0,r=new Array(t);n=e.length?{done:!0}:{done:!1,value:e[r++]}},e:function(e){throw e},f:i}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var a,s=!0,o=!1;return{s:function(){n=n.call(e)},n:function(){var e=n.next();return s=e.done,e},e:function(e){o=!0,a=e},f:function(){try{s||null==n.return||n.return()}finally{if(o)throw a}}}}var v,m;function k(e){if(v){for(var t=arguments.length,n=new Array(t>1?t-1:0),r=1;r1?t-1:0),r=1;r>5,this.ntype=31&this.payload[0],this.isvcl=1==this.ntype||5==this.ntype,this.stype="",this.isfmb=!1}return i(e,[{key:"toString",value:function(){return"".concat(e.type(this),": NRI: ").concat(this.getNri())}},{key:"getNri",value:function(){return this.nri}},{key:"type",value:function(){return this.ntype}},{key:"isKeyframe",value:function(){return this.ntype===e.IDR}},{key:"getPayload",value:function(){return this.payload}},{key:"getPayloadSize",value:function(){return this.payload.byteLength}},{key:"getSize",value:function(){return 4+this.getPayloadSize()}},{key:"getData",value:function(){var e=new Uint8Array(this.getSize());return new DataView(e.buffer).setUint32(0,this.getSize()-4),e.set(this.getPayload(),4),e}}],[{key:"NDR",get:function(){return 1}},{key:"IDR",get:function(){return 5}},{key:"SEI",get:function(){return 6}},{key:"SPS",get:function(){return 7}},{key:"PPS",get:function(){return 8}},{key:"AUD",get:function(){return 9}},{key:"TYPES",get:function(){var t;return a(t={},e.IDR,"IDR"),a(t,e.SEI,"SEI"),a(t,e.SPS,"SPS"),a(t,e.PPS,"PPS"),a(t,e.NDR,"NDR"),a(t,e.AUD,"AUD"),t}},{key:"type",value:function(t){return t.ntype in e.TYPES?e.TYPES[t.ntype]:"UNKNOWN"}}]),e}();function S(e,t){var n=new Uint8Array((0|e.byteLength)+(0|t.byteLength));return n.set(e,0),n.set(t,0|e.byteLength),n}var w=function(){function e(t){n(this,e),this.data=t,this.index=0,this.bitLength=8*t.byteLength}return i(e,[{key:"setData",value:function(e){this.data=e,this.index=0,this.bitLength=8*e.byteLength}},{key:"bitsAvailable",get:function(){return this.bitLength-this.index}},{key:"skipBits",value:function(e){if(this.bitsAvailable1&&void 0!==arguments[1])||arguments[1],n=this.getBits(e,this.index,t);return n}},{key:"getBits",value:function(e,t){var n=!(arguments.length>2&&void 0!==arguments[2])||arguments[2];if(this.bitsAvailable>>r,a=8-r;if(a>=e)return n&&(this.index+=e),i>>a-e;n&&(this.index+=a);var s=e-a;return i<>>1:-1*(e>>>1)}},{key:"readBoolean",value:function(){return 1===this.readBits(1)}},{key:"readUByte",value:function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;return this.readBits(8*e)}},{key:"readUShort",value:function(){return this.readBits(16)}},{key:"readUInt",value:function(){return this.readBits(32)}}]),e}(),x=function(){function e(t){n(this,e),this.remuxer=t,this.track=t.mp4track}return i(e,[{key:"parseSPS",value:function(t){var n=e.readSPS(new Uint8Array(t));this.track.fps=n.fps,this.track.width=n.width,this.track.height=n.height,this.track.sps=[new Uint8Array(t)],this.track.codec="avc1.";for(var r=new DataView(t.buffer,t.byteOffset+1,4),i=0;i<3;++i){var a=r.getUint8(i).toString(16);a.length<2&&(a="0"+a),this.track.codec+=a}}},{key:"parsePPS",value:function(e){this.track.pps=[new Uint8Array(e)]}},{key:"parseNAL",value:function(e){if(!e)return!1;var t=!1;switch(e.type()){case b.IDR:case b.NDR:t=!0;break;case b.PPS:this.track.pps||(this.parsePPS(e.getPayload()),!this.remuxer.readyToDecode&&this.track.pps&&this.track.sps&&(this.remuxer.readyToDecode=!0)),t=!0;break;case b.SPS:this.track.sps||(this.parseSPS(e.getPayload()),!this.remuxer.readyToDecode&&this.track.pps&&this.track.sps&&(this.remuxer.readyToDecode=!0)),t=!0;break;case b.AUD:k("AUD - ignoing");break;case b.SEI:k("SEI - ignoing")}return t}}],[{key:"extractNALu",value:function(e){for(var t,n,r=0,i=e.byteLength,a=0,s=[],o=0;r0&&x[1]>0&&(h=x[0]/x[1])}if(u.readBoolean()&&u.skipBits(1),u.readBoolean()&&(u.skipBits(4),u.readBoolean()&&u.skipBits(24)),u.readBoolean()&&(u.skipUEG(),u.skipUEG()),u.readBoolean()){var A=u.readUInt(),U=u.readUInt();u.readBoolean()&&(p=U/(2*A))}}return{fps:p>0?p:void 0,width:Math.ceil((16*(i+1)-2*c-2*f)*h),height:(2-s)*(a+1)*16-(s?2:4)*(l+d)}}},{key:"parseHeader",value:function(e){var t=new w(e.getPayload());t.readUByte(),e.isfmb=0===t.readUEG(),e.stype=t.readUEG()}}]),e}(),A=function(){function e(t){n(this,e),this.remuxer=t,this.track=t.mp4track}return i(e,[{key:"setAACConfig",value:function(){var t,n,r,i=new Uint8Array(2),a=e.aacHeader;a&&(t=1+((192&a[2])>>>6),n=(60&a[2])>>>2,r=(1&a[2])<<2,r|=(192&a[3])>>>6,i[0]=t<<3,i[0]|=(14&n)>>1,i[1]|=(1&n)<<7,i[1]|=r<<3,this.track.codec="mp4a.40."+t,this.track.channelCount=r,this.track.config=i,this.remuxer.readyToDecode=!0)}}],[{key:"samplingRateMap",get:function(){return[96e3,88200,64e3,48e3,44100,32e3,24e3,22050,16e3,12e3,11025,8e3,7350]}},{key:"getHeaderLength",value:function(e){return 1&e[1]?7:9}},{key:"getFrameLength",value:function(e){return(3&e[3])<<11|e[4]<<3|(224&e[5])>>>5}},{key:"isAACPattern",value:function(e){return 255===e[0]&&240==(240&e[1])&&0==(6&e[1])}},{key:"extractAAC",value:function(t){var n,r,i=0,a=t.byteLength,s=[];if(!e.isAACPattern(t))return g("Invalid ADTS audio format"),s;for(n=e.getHeaderLength(t),e.aacHeader||(e.aacHeader=t.subarray(0,n));i-1&&this.listener[e].splice(n,1),!0}return!1}},{key:"offAll",value:function(){this.listener={}}},{key:"dispatch",value:function(e,t){return!!this.listener[e]&&(this.listener[e].map((function(e){e.apply(null,[t])})),!0)}}]),e}(),B=function(){function e(){n(this,e)}return i(e,null,[{key:"init",value:function(){var t;for(t in e.types={avc1:[],avcC:[],btrt:[],dinf:[],dref:[],esds:[],ftyp:[],hdlr:[],mdat:[],mdhd:[],mdia:[],mfhd:[],minf:[],moof:[],moov:[],mp4a:[],mvex:[],mvhd:[],sdtp:[],stbl:[],stco:[],stsc:[],stsd:[],stsz:[],stts:[],tfdt:[],tfhd:[],traf:[],trak:[],trun:[],trex:[],tkhd:[],vmhd:[],smhd:[]},e.types)e.types.hasOwnProperty(t)&&(e.types[t]=[t.charCodeAt(0),t.charCodeAt(1),t.charCodeAt(2),t.charCodeAt(3)]);var n=new Uint8Array([0,0,0,0,0,0,0,0,118,105,100,101,0,0,0,0,0,0,0,0,0,0,0,0,86,105,100,101,111,72,97,110,100,108,101,114,0]),r=new Uint8Array([0,0,0,0,0,0,0,0,115,111,117,110,0,0,0,0,0,0,0,0,0,0,0,0,83,111,117,110,100,72,97,110,100,108,101,114,0]);e.HDLR_TYPES={video:n,audio:r};var i=new Uint8Array([0,0,0,0,0,0,0,1,0,0,0,12,117,114,108,32,0,0,0,1]),a=new Uint8Array([0,0,0,0,0,0,0,0]);e.STTS=e.STSC=e.STCO=a,e.STSZ=new Uint8Array([0,0,0,0,0,0,0,0,0,0,0,0]),e.VMHD=new Uint8Array([0,0,0,1,0,0,0,0,0,0,0,0]),e.SMHD=new Uint8Array([0,0,0,0,0,0,0,0]),e.STSD=new Uint8Array([0,0,0,0,0,0,0,1]);var s=new Uint8Array([105,115,111,109]),o=new Uint8Array([97,118,99,49]),u=new Uint8Array([0,0,0,1]);e.FTYP=e.box(e.types.ftyp,s,u,s,o),e.DINF=e.box(e.types.dinf,e.box(e.types.dref,i))}},{key:"box",value:function(e){for(var t=arguments.length,n=new Array(t>1?t-1:0),r=1;r>24&255,i[1]=a>>16&255,i[2]=a>>8&255,i[3]=255&a,i.set(e,4),s=0,a=8;s>24&255,t>>16&255,t>>8&255,255&t,n>>24,n>>16&255,n>>8&255,255&n,85,196,0,0]))}},{key:"mdia",value:function(t){return e.box(e.types.mdia,e.mdhd(t.timescale,t.duration),e.hdlr(t.type),e.minf(t))}},{key:"mfhd",value:function(t){return e.box(e.types.mfhd,new Uint8Array([0,0,0,0,t>>24,t>>16&255,t>>8&255,255&t]))}},{key:"minf",value:function(t){return"audio"===t.type?e.box(e.types.minf,e.box(e.types.smhd,e.SMHD),e.DINF,e.stbl(t)):e.box(e.types.minf,e.box(e.types.vmhd,e.VMHD),e.DINF,e.stbl(t))}},{key:"moof",value:function(t,n,r){return e.box(e.types.moof,e.mfhd(t),e.traf(r,n))}},{key:"moov",value:function(t,n,r){for(var i=t.length,a=[];i--;)a[i]=e.trak(t[i]);return e.box.apply(null,[e.types.moov,e.mvhd(r,n)].concat(a).concat(e.mvex(t)))}},{key:"mvex",value:function(t){for(var n=t.length,r=[];n--;)r[n]=e.trex(t[n]);return e.box.apply(null,[e.types.mvex].concat(r))}},{key:"mvhd",value:function(t,n){var r=new Uint8Array([0,0,0,0,0,0,0,1,0,0,0,2,t>>24&255,t>>16&255,t>>8&255,255&t,n>>24&255,n>>16&255,n>>8&255,255&n,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255]);return e.box(e.types.mvhd,r)}},{key:"sdtp",value:function(t){var n,r,i=t.samples||[],a=new Uint8Array(4+i.length);for(r=0;r>>8&255),a.push(255&i),a=a.concat(Array.prototype.slice.call(r));for(n=0;n>>8&255),s.push(255&i),s=s.concat(Array.prototype.slice.call(r));var o=e.box(e.types.avcC,new Uint8Array([1,a[3],a[4],a[5],255,224|t.sps.length].concat(a).concat([t.pps.length]).concat(s))),u=t.width,c=t.height;return e.box(e.types.avc1,new Uint8Array([0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,u>>8&255,255&u,c>>8&255,255&c,0,72,0,0,0,72,0,0,0,0,0,0,0,1,18,98,105,110,101,108,112,114,111,46,114,117,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,24,17,17]),o,e.box(e.types.btrt,new Uint8Array([0,28,156,128,0,45,198,192,0,45,198,192])))}},{key:"esds",value:function(e){var t=e.config.byteLength,n=new Uint8Array(26+t+3);return n.set([0,0,0,0,3,23+t,0,1,0,4,15+t,64,21,0,0,0,0,0,0,0,0,0,0,0,5,t]),n.set(e.config,26),n.set([6,1,2],26+t),n}},{key:"mp4a",value:function(t){var n=t.audiosamplerate;return e.box(e.types.mp4a,new Uint8Array([0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,t.channelCount,0,16,0,0,0,0,n>>8&255,255&n,0,0]),e.box(e.types.esds,e.esds(t)))}},{key:"stsd",value:function(t){return"audio"===t.type?e.box(e.types.stsd,e.STSD,e.mp4a(t)):e.box(e.types.stsd,e.STSD,e.avc1(t))}},{key:"tkhd",value:function(t){var n=t.id,r=t.duration,i=t.width,a=t.height,s=t.volume;return e.box(e.types.tkhd,new Uint8Array([0,0,0,7,0,0,0,0,0,0,0,0,n>>24&255,n>>16&255,n>>8&255,255&n,0,0,0,0,r>>24,r>>16&255,r>>8&255,255&r,0,0,0,0,0,0,0,0,0,0,0,0,s>>0&255,s%1*10>>0&255,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,0,0,0,i>>8&255,255&i,0,0,a>>8&255,255&a,0,0]))}},{key:"traf",value:function(t,n){var r=e.sdtp(t),i=t.id;return e.box(e.types.traf,e.box(e.types.tfhd,new Uint8Array([0,0,0,0,i>>24,i>>16&255,i>>8&255,255&i])),e.box(e.types.tfdt,new Uint8Array([0,0,0,0,n>>24,n>>16&255,n>>8&255,255&n])),e.trun(t,r.length+16+16+8+16+8+8),r)}},{key:"trak",value:function(t){return t.duration=t.duration||4294967295,e.box(e.types.trak,e.tkhd(t),e.mdia(t))}},{key:"trex",value:function(t){var n=t.id;return e.box(e.types.trex,new Uint8Array([0,0,0,0,n>>24,n>>16&255,n>>8&255,255&n,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1]))}},{key:"trun",value:function(t,n){var r,i,a,s,o,u,c=t.samples||[],f=c.length,l=12+16*f,d=new Uint8Array(l);for(n+=8+l,d.set([0,0,15,1,f>>>24&255,f>>>16&255,f>>>8&255,255&f,n>>>24&255,n>>>16&255,n>>>8&255,255&n],0),r=0;r>>24&255,a>>>16&255,a>>>8&255,255&a,s>>>24&255,s>>>16&255,s>>>8&255,255&s,o.isLeading<<2|o.dependsOn,o.isDependedOn<<6|o.hasRedundancy<<4|o.paddingValue<<1|o.isNonSync,61440&o.degradPrio,15&o.degradPrio,u>>>24&255,u>>>16&255,u>>>8&255,255&u],12+16*r);return e.box(e.types.trun,d)}},{key:"initSegment",value:function(t,n,r){e.types||e.init();var i,a=e.moov(t,n,r);return(i=new Uint8Array(e.FTYP.byteLength+a.byteLength)).set(e.FTYP),i.set(a,e.FTYP.byteLength),i}}]),e}(),D=1,C=function(){function e(){n(this,e)}return i(e,[{key:"flush",value:function(){this.mp4track.len=0,this.mp4track.samples=[]}},{key:"isReady",value:function(){return!(!this.readyToDecode||!this.samples.length)||null}}],[{key:"getTrackID",value:function(){return D++}}]),e}(),E=function(e){s(r,e);var t=l(r);function r(e){var i;return n(this,r),(i=t.call(this)).readyToDecode=!1,i.nextDts=0,i.dts=0,i.mp4track={id:C.getTrackID(),type:"audio",channelCount:0,len:0,fragmented:!0,timescale:e,duration:e,samples:[],config:"",codec:""},i.samples=[],i.aac=new A(c(i)),i}return i(r,[{key:"resetTrack",value:function(){this.readyToDecode=!1,this.mp4track.codec="",this.mp4track.channelCount="",this.mp4track.config="",this.mp4track.timescale=this.timescale,this.nextDts=0,this.dts=0}},{key:"remux",value:function(e){if(e.length>0)for(var t=0;t0&&this.readyToDecode&&(this.mp4track.len+=s,this.samples.push({units:a,size:s,keyFrame:i.keyFrame,duration:i.duration,compositionTimeOffset:i.compositionTimeOffset}))}}catch(e){n.e(e)}finally{n.f()}}},{key:"getPayload",value:function(){if(!this.isReady())return null;var e,t,n=new Uint8Array(this.mp4track.len),r=0,i=this.mp4track.samples;for(this.dts=this.nextDts;this.samples.length;){var a=this.samples.shift(),s=a.units;if((t=a.duration)<=0)k("remuxer: invalid sample duration at DTS: ".concat(this.nextDts," :").concat(t)),this.mp4track.len-=a.size;else{this.nextDts+=t,e={size:a.size,duration:t,cts:a.compositionTimeOffset||0,flags:{isLeading:0,isDependedOn:0,hasRedundancy:0,degradPrio:0,isNonSync:a.keyFrame?0:1,dependsOn:a.keyFrame?2:1}};var o,u=y(s);try{for(u.s();!(o=u.n()).done;){var c=o.value;n.set(c.getData(),r),r+=c.getSize()}}catch(e){u.e(e)}finally{u.f()}i.push(e)}}return i.length?new Uint8Array(n.buffer,0,this.mp4track.len):null}}]),r}(C),P=function(e){s(r,e);var t=l(r);function r(e){var i;return n(this,r),(i=t.call(this,"remuxer")).initialized=!1,i.trackTypes=[],i.tracks={},i.seq=1,i.env=e,i.timescale=1e3,i.mediaDuration=0,i.aacParser=null,i}return i(r,[{key:"addTrack",value:function(e){if("video"!==e&&"both"!==e||(this.tracks.video=new T(this.timescale),this.trackTypes.push("video")),"audio"===e||"both"===e){var t=new E(this.timescale);this.aacParser=t.getAacParser(),this.tracks.audio=t,this.trackTypes.push("audio")}}},{key:"reset",value:function(){var e,t=y(this.trackTypes);try{for(t.s();!(e=t.n()).done;){var n=e.value;this.tracks[n].resetTrack()}}catch(e){t.e(e)}finally{t.f()}this.initialized=!1}},{key:"destroy",value:function(){this.tracks={},this.offAll()}},{key:"flush",value:function(){if(this.initialized){var e,t=y(this.trackTypes);try{for(t.s();!(e=t.n()).done;){var n=e.value,r=this.tracks[n],i=r.getPayload();if(i&&i.byteLength){var a={type:n,payload:S(B.moof(this.seq,r.dts,r.mp4track),B.mdat(i)),dts:r.dts};"video"===n&&(a.fps=r.mp4track.fps),this.dispatch("buffer",a);var s=(o=r.dts/this.timescale,u=void 0,c=void 0,f=void 0,l=void 0,l="",u=Math.floor(o),(c=parseInt(u/3600,10)%24)>0&&(l+=(c<10?"0"+c:c)+":"),l+=((f=parseInt(u/60,10)%60)<10?"0"+f:f)+":"+((u=u<0?0:u%60)<10?"0"+u:u));k("put segment (".concat(n,"): dts: ").concat(r.dts," frames: ").concat(r.mp4track.samples.length," second: ").concat(s)),r.flush(),this.seq++}}}catch(e){t.e(e)}finally{t.f()}}else this.isReady()&&(this.dispatch("ready"),this.initSegment(),this.initialized=!0,this.flush());var o,u,c,f,l}},{key:"initSegment",value:function(){var e,t=[],n=y(this.trackTypes);try{for(n.s();!(e=n.n()).done;){var r=e.value,i=this.tracks[r];if("browser"==this.env){var a={type:r,payload:B.initSegment([i.mp4track],this.mediaDuration,this.timescale)};this.dispatch("buffer",a)}else t.push(i.mp4track)}}catch(e){n.e(e)}finally{n.f()}if("node"==this.env){var s={type:"all",payload:B.initSegment(t,this.mediaDuration,this.timescale)};this.dispatch("buffer",s)}k("Initial segment generated.")}},{key:"isReady",value:function(){var e,t=y(this.trackTypes);try{for(t.s();!(e=t.n()).done;){var n=e.value;if(!this.tracks[n].readyToDecode||!this.tracks[n].samples.length)return!1}}catch(e){t.e(e)}finally{t.f()}return!0}},{key:"remux",value:function(e){var t,n=y(this.trackTypes);try{for(n.s();!(t=n.n()).done;){var r=t.value,i=e[r];"audio"===r&&this.tracks.video&&!this.tracks.video.readyToDecode||i.length>0&&this.tracks[r].remux(i)}}catch(e){n.e(e)}finally{n.f()}this.flush()}}]),r}(U),L=function(e){s(r,e);var t=l(r);function r(e,i){var a;return n(this,r),(a=t.call(this,"buffer")).type=i,a.queue=new Uint8Array,a.cleaning=!1,a.pendingCleaning=0,a.cleanOffset=30,a.cleanRanges=[],a.sourceBuffer=e,a.sourceBuffer.addEventListener("updateend",(function(){a.pendingCleaning>0&&(a.initCleanup(a.pendingCleaning),a.pendingCleaning=0),a.cleaning=!1,a.cleanRanges.length&&a.doCleanup()})),a.sourceBuffer.addEventListener("error",(function(){a.dispatch("error",{type:a.type,name:"buffer",error:"buffer error"})})),a}return i(r,[{key:"destroy",value:function(){this.queue=null,this.sourceBuffer=null,this.offAll()}},{key:"doCleanup",value:function(){if(this.cleanRanges.length){var e=this.cleanRanges.shift();k("".concat(this.type," remove range [").concat(e[0]," - ").concat(e[1],")")),this.cleaning=!0,this.sourceBuffer.remove(e[0],e[1])}else this.cleaning=!1}},{key:"initCleanup",value:function(e){try{if(this.sourceBuffer.updating)return void(this.pendingCleaning=e);if(this.sourceBuffer.buffered&&this.sourceBuffer.buffered.length&&!this.cleaning){for(var t=0;tthis.cleanOffset&&n<(r=e-this.cleanOffset)&&this.cleanRanges.push([n,r])}this.doCleanup()}}catch(e){g("Error occured while cleaning ".concat(this.type," buffer - ").concat(e.name,": ").concat(e.message))}}},{key:"doAppend",value:function(){if(this.queue.length&&this.sourceBuffer&&!this.sourceBuffer.updating)try{this.sourceBuffer.appendBuffer(this.queue),this.queue=new Uint8Array}catch(t){var e="unexpectedError";"QuotaExceededError"===t.name?(k("".concat(this.type," buffer quota full")),e="QuotaExceeded"):(g("Error occured while appending ".concat(this.type," buffer - ").concat(t.name,": ").concat(t.message)),e="InvalidStateError"),this.dispatch("error",{type:this.type,name:e,error:"buffer error"})}}},{key:"feed",value:function(e){this.queue=S(this.queue,e)}}]),r}(U);return function(r){s(o,r);var a=l(o);function o(e){var r;n(this,o),(r=a.call(this,"jmuxer")).isReset=!1;return r.options=Object.assign({},{node:"",mode:"both",flushingTime:500,maxDelay:500,clearBuffer:!0,fps:30,readFpsFromTrack:!1,debug:!1,onReady:function(){},onData:function(){},onError:function(){},onMissingVideoFrames:function(){},onMissingAudioFrames:function(){}},e),r.env="object"===("undefined"==typeof process?"undefined":t(process))&&"undefined"==typeof window?"node":"browser",r.options.debug&&(v=console.log,m=console.error),r.options.fps||(r.options.fps=30),r.frameDuration=1e3/r.options.fps|0,r.remuxController=new P(r.env),r.remuxController.addTrack(r.options.mode),r.initData(),r.remuxController.on("buffer",r.onBuffer.bind(c(r))),"browser"==r.env&&(r.remuxController.on("ready",r.createBuffer.bind(c(r))),r.initBrowser()),r}return i(o,[{key:"initData",value:function(){this.lastCleaningTime=Date.now(),this.kfPosition=[],this.kfCounter=0,this.pendingUnits={},this.remainingData=new Uint8Array,this.startInterval()}},{key:"initBrowser",value:function(){"string"==typeof this.options.node&&""==this.options.node&&g("no video element were found to render, provide a valid video element"),this.node="string"==typeof this.options.node?document.getElementById(this.options.node):this.options.node,this.mseReady=!1,this.setupMSE()}},{key:"createStream",value:function(){var t=this.feed.bind(this),n=this.destroy.bind(this);return this.stream=new e.Duplex({writableObjectMode:!0,read:function(e){},write:function(e,n,r){t(e),r()},final:function(e){n(),e()}}),this.stream}},{key:"setupMSE",value:function(){if(window.MediaSource=window.MediaSource||window.WebKitMediaSource||window.ManagedMediaSource,!window.MediaSource)throw"Oops! Browser does not support Media Source Extension or Managed Media Source (IOS 17+).";if(this.isMSESupported=!!window.MediaSource,this.mediaSource=new window.MediaSource,this.url=URL.createObjectURL(this.mediaSource),window.MediaSource===window.ManagedMediaSource)try{this.node.removeAttribute("src"),this.node.disableRemotePlayback=!0;var e=document.createElement("source");e.type="video/mp4",e.src=this.url,this.node.appendChild(e),this.node.load()}catch(e){this.node.src=this.url}else this.node.src=this.url;this.mseEnded=!1,this.mediaSource.addEventListener("sourceopen",this.onMSEOpen.bind(this)),this.mediaSource.addEventListener("sourceclose",this.onMSEClose.bind(this)),this.mediaSource.addEventListener("webkitsourceopen",this.onMSEOpen.bind(this)),this.mediaSource.addEventListener("webkitsourceclose",this.onMSEClose.bind(this))}},{key:"endMSE",value:function(){if(!this.mseEnded)try{this.mseEnded=!0,this.mediaSource.endOfStream()}catch(e){g("mediasource is not available to end")}}},{key:"feed",value:function(e){var t,n,r,i=!1,a={video:[],audio:[]};if(e&&this.remuxController){if(r=e.duration?parseInt(e.duration):0,e.video){e.video=S(this.remainingData,e.video);var s=d(x.extractNALu(e.video),2);if(t=s[0],n=s[1],this.remainingData=n||new Uint8Array,!(t.length>0))return g("Failed to extract any NAL units from video data:",n),void("function"==typeof this.options.onMissingVideoFrames&&this.options.onMissingVideoFrames.call(null,e));a.video=this.getVideoFrames(t,r,e.compositionTimeOffset),i=!0}if(e.audio){if(!((t=A.extractAAC(e.audio)).length>0))return g("Failed to extract audio data from:",e.audio),void("function"==typeof this.options.onMissingAudioFrames&&this.options.onMissingAudioFrames.call(null,e));a.audio=this.getAudioFrames(t,r),i=!0}i?this.remuxController.remux(a):g("Input object must have video and/or audio property. Make sure it is a valid typed array")}}},{key:"getVideoFrames",value:function(e,t,n){var r,i=this,a=[],s=[],o=0,u=!1,c=!1;this.pendingUnits.units&&(a=this.pendingUnits.units,c=this.pendingUnits.vcl,u=this.pendingUnits.keyFrame,this.pendingUnits={});var f,l=y(e);try{for(l.s();!(f=l.n()).done;){var d=f.value,h=new b(d);h.type()!==b.IDR&&h.type()!==b.NDR||x.parseHeader(h),a.length&&c&&(h.isfmb||!h.isvcl)&&(s.push({units:a,keyFrame:u}),a=[],u=!1,c=!1),a.push(h),u=u||h.isKeyframe(),c=c||h.isvcl}}catch(e){l.e(e)}finally{l.f()}if(a.length)if(t)if(c)s.push({units:a,keyFrame:u});else{var p=s.length-1;p>=0&&(s[p].units=s[p].units.concat(a))}else this.pendingUnits={units:a,keyFrame:u,vcl:c};return r=t?t/s.length|0:this.frameDuration,o=t?t-r*s.length:0,s.map((function(e){e.duration=r,e.compositionTimeOffset=n,o>0&&(e.duration++,o--),i.kfCounter++,e.keyFrame&&i.options.clearBuffer&&i.kfPosition.push(i.kfCounter*r/1e3)})),k("jmuxer: No. of frames of the last chunk: ".concat(s.length)),s}},{key:"getAudioFrames",value:function(e,t){var n,r,i=[],a=0,s=y(e);try{for(s.s();!(r=s.n()).done;){var o=r.value;i.push({units:o})}}catch(e){s.e(e)}finally{s.f()}return n=t?t/i.length|0:this.frameDuration,a=t?t-n*i.length:0,i.map((function(e){e.duration=n,a>0&&(e.duration++,a--)})),i}},{key:"destroy",value:function(){if(this.stopInterval(),this.stream&&(this.remuxController.flush(),this.stream.push(null),this.stream=null),this.remuxController&&(this.remuxController.destroy(),this.remuxController=null),this.bufferControllers){for(var e in this.bufferControllers)this.bufferControllers[e].destroy();this.bufferControllers=null,this.endMSE()}this.node=!1,this.mseReady=!1,this.videoStarted=!1,this.mediaSource=null}},{key:"reset",value:function(){if(this.stopInterval(),this.isReset=!0,this.node.pause(),this.remuxController&&this.remuxController.reset(),this.bufferControllers){for(var e in this.bufferControllers)this.bufferControllers[e].destroy();this.bufferControllers=null,this.endMSE()}this.initData(),"browser"==this.env&&this.initBrowser(),k("JMuxer was reset")}},{key:"createBuffer",value:function(){if(this.mseReady&&this.remuxController&&this.remuxController.isReady()&&!this.bufferControllers)for(var e in this.bufferControllers={},this.remuxController.tracks){var t=this.remuxController.tracks[e];if(!o.isSupported("".concat(e,'/mp4; codecs="').concat(t.mp4track.codec,'"')))return g("Browser does not support codec"),!1;var n=this.mediaSource.addSourceBuffer("".concat(e,'/mp4; codecs="').concat(t.mp4track.codec,'"'));this.bufferControllers[e]=new L(n,e),this.bufferControllers[e].on("error",this.onBufferError.bind(this))}}},{key:"startInterval",value:function(){var e=this;this.interval=setInterval((function(){e.options.flushingTime?e.applyAndClearBuffer():e.bufferControllers&&e.cancelDelay()}),this.options.flushingTime||1e3)}},{key:"stopInterval",value:function(){this.interval&&clearInterval(this.interval)}},{key:"cancelDelay",value:function(){if(this.node.buffered&&this.node.buffered.length>0&&!this.node.seeking){var e=this.node.buffered.end(0);e-this.node.currentTime>this.options.maxDelay/1e3&&(console.log("delay"),this.node.currentTime=e-.001)}}},{key:"releaseBuffer",value:function(){for(var e in this.bufferControllers)this.bufferControllers[e].doAppend()}},{key:"applyAndClearBuffer",value:function(){this.bufferControllers&&(this.releaseBuffer(),this.clearBuffer())}},{key:"getSafeClearOffsetOfBuffer",value:function(e){for(var t,n="audio"===this.options.mode&&e||0,r=0;r=e);r++)t=this.kfPosition[r];return t&&(this.kfPosition=this.kfPosition.filter((function(e){return e=t}))),n}},{key:"clearBuffer",value:function(){if(this.options.clearBuffer&&Date.now()-this.lastCleaningTime>1e4){for(var e in this.bufferControllers){var t=this.getSafeClearOffsetOfBuffer(this.node.currentTime);this.bufferControllers[e].initCleanup(t)}this.lastCleaningTime=Date.now()}}},{key:"onBuffer",value:function(e){this.options.readFpsFromTrack&&void 0!==e.fps&&this.options.fps!=e.fps&&(this.options.fps=e.fps,this.frameDuration=Math.ceil(1e3/e.fps),k("JMuxer changed FPS to ".concat(e.fps," from track data"))),"browser"==this.env?this.bufferControllers&&this.bufferControllers[e.type]&&this.bufferControllers[e.type].feed(e.payload):this.stream&&this.stream.push(e.payload),this.options.onData&&this.options.onData(e.payload),0===this.options.flushingTime&&this.applyAndClearBuffer()}},{key:"onMSEOpen",value:function(){this.mseReady=!0,URL.revokeObjectURL(this.url),"function"==typeof this.options.onReady&&this.options.onReady.call(null,this.isReset)}},{key:"onMSEClose",value:function(){this.mseReady=!1,this.videoStarted=!1}},{key:"onBufferError",value:function(e){if("QuotaExceeded"==e.name)return k("JMuxer cleaning ".concat(e.type," buffer due to QuotaExceeded error")),void this.bufferControllers[e.type].initCleanup(this.node.currentTime);"InvalidStateError"==e.name?(k("JMuxer is reseting due to InvalidStateError"),this.reset()):this.endMSE(),"function"==typeof this.options.onError&&this.options.onError.call(null,e)}}],[{key:"isSupported",value:function(e){return window.MediaSource&&window.MediaSource.isTypeSupported(e)}}]),o}(U)})); diff --git a/ios/Runner/AppDelegate.m b/ios/Runner/AppDelegate.m index 76d97654..b648e544 100755 --- a/ios/Runner/AppDelegate.m +++ b/ios/Runner/AppDelegate.m @@ -76,12 +76,25 @@ /* * 苹果推送注册成功回调，将苹果返回的deviceToken上传到CloudPush服务器 */ -- (void)application:(UIApplication *)application didRegisterForRemoteNotificationsWithDeviceToken:(NSData *)deviceToken { +//- (void)application:(UIApplication *)application didRegisterForRemoteNotificationsWithDeviceToken:(NSData *)deviceToken { +// NSString *tokenString = [self hexStringFromData:deviceToken]; +// NSLog(@"starlock didRegisterForRemoteNotificationsWithDeviceToken token: %@", tokenString); +// /// Required - 注册 DeviceToken +// [JPUSHService registerDeviceToken:deviceToken]; +// +//} - /// Required - 注册 DeviceToken - [JPUSHService registerDeviceToken:deviceToken]; +- (NSString *)hexStringFromData:(NSData *)data { + const unsigned char *dataBuffer = (const unsigned char *)[data bytes]; + NSMutableString *hexString = [NSMutableString stringWithCapacity:data.length * 2]; + for (NSInteger i = 0; i < data.length; i++) { + [hexString appendFormat:@"%02x", dataBuffer[i]]; + } + + return [hexString copy]; } + /* * 苹果推送注册失败回调 */ diff --git a/lib/appRouters.dart b/lib/appRouters.dart index c7d403a1..e1fa7672 100755 --- a/lib/appRouters.dart +++ b/lib/appRouters.dart @@ -61,6 +61,7 @@ import 'package:star_lock/mine/valueAddedServices/advancedFeaturesWeb/advancedFe import 'package:star_lock/mine/valueAddedServices/advancedFunctionRecord/advancedFunctionRecord_page.dart'; import 'package:star_lock/mine/valueAddedServices/valueAddedServicesRecord/value_added_services_record_page.dart'; import 'package:star_lock/talk/starChart/views/talkView/talk_view_page.dart'; +import 'package:star_lock/talk/starChart/webView/h264_web_view.dart'; import 'common/safetyVerification/safetyVerification_page.dart'; import 'login/forgetPassword/starLock_forgetPassword_page.dart'; @@ -515,6 +516,7 @@ abstract class Routers { static const String doubleLockLinkPage = '/doubleLockLinkPage'; //双锁联动 static const String starChartPage = '/starChartPage'; //星图 static const String starChartTalkView = '/starChartTalkView'; //星图对讲页面 + static const String h264WebView = '/h264WebView'; //星图对讲页面 } abstract class AppRouters { @@ -1195,5 +1197,6 @@ abstract class AppRouters { page: () => const DoubleLockLinkPage()), GetPage( name: Routers.starChartTalkView, page: () => const TalkViewPage()), + GetPage(name: Routers.h264WebView, page: () => H264WebView()), ]; } diff --git a/lib/main/lockDetail/doorLockLog/doorLockLog_page.dart b/lib/main/lockDetail/doorLockLog/doorLockLog_page.dart index 3a5bcdd8..51b74482 100755 --- a/lib/main/lockDetail/doorLockLog/doorLockLog_page.dart +++ b/lib/main/lockDetail/doorLockLog/doorLockLog_page.dart @@ -267,26 +267,31 @@ class _DoorLockLogPageState extends State with RouteAware { color: Colors.white, borderRadius: BorderRadius.circular(16.w), ), - child: Obx( - () => state.lockLogItemList.isNotEmpty - ? Timeline.tileBuilder( - builder: _timelineBuilderWidget(), - theme: TimelineThemeData( - nodePosition: 0.04, //居左侧距离 - connectorTheme: const ConnectorThemeData( - thickness: 1.0, - color: AppColors.greyLineColor, - indent: 0.5, + child: Obx(() => EasyRefreshTool( + onRefresh: () async { + logic.mockNetworkDataRequest(isRefresh: true); + }, + onLoad: () async { + logic.mockNetworkDataRequest(isRefresh: false); + }, + child: state.lockLogItemList.isNotEmpty + ? Timeline.tileBuilder( + builder: _timelineBuilderWidget(), + theme: TimelineThemeData( + nodePosition: 0.04, //居左侧距离 + connectorTheme: const ConnectorThemeData( + thickness: 1.0, + color: AppColors.greyLineColor, + indent: 0.5, + ), + indicatorTheme: const IndicatorThemeData( + size: 8.0, + color: AppColors.greyLineColor, + position: 0.4, + ), ), - indicatorTheme: const IndicatorThemeData( - size: 8.0, - color: AppColors.greyLineColor, - position: 0.45, - ), - ), - ) - : NoData(), - ), + ) + : NoData())), ); } @@ -347,6 +352,9 @@ class _DoorLockLogPageState extends State with RouteAware { if (recordData.videoUrl != null && recordData.videoUrl!.isNotEmpty) { final lockLogItemList = state.lockLogItemList.value; final list = lockLogItemList + .where((e) => + (e.videoUrl != null && e.videoUrl!.isNotEmpty) || + (e.imagesUrl != null && e.imagesUrl!.isNotEmpty)) .map( (e) => RecordListData( videoUrl: e.videoUrl, diff --git a/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart b/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart index c5b6b7cd..a95b1c78 100644 --- a/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart +++ b/lib/talk/starChart/handle/impl/udp_talk_data_handler.dart @@ -2,6 +2,7 @@ import 'dart:typed_data'; import 'package:star_lock/app_settings/app_settings.dart'; import 'package:star_lock/talk/starChart/constant/message_type_constant.dart'; import 'package:star_lock/talk/starChart/entity/scp_message.dart'; +import 'package:star_lock/talk/starChart/handle/other/h264_frame_handler.dart'; import 'package:star_lock/talk/starChart/handle/scp_message_base_handle.dart'; import 'package:star_lock/talk/starChart/handle/scp_message_handle.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; @@ -158,16 +159,13 @@ class UdpTalkDataHandler extends ScpMessageBaseHandle void _handleVideoH264(TalkData talkData) { final TalkDataH264Frame talkDataH264Frame = TalkDataH264Frame(); talkDataH264Frame.mergeFromBuffer(talkData.content); - // AppLog.log('H264 TalkData :$talkDataH264Frame'); - // talkDataRepository.addTalkData(talkData); + frameHandler.handleFrame(talkDataH264Frame); } /// 处理图片数据 void _handleVideoImage(TalkData talkData) async { final List processCompletePayload = await _processCompletePayload(Uint8List.fromList(talkData.content)); - // AppLog.log('得到完整的帧：${processCompletePayload.length}'); // 循环发送每一帧的数据 - processCompletePayload.forEach((element) { talkData.content = element; talkDataRepository.addTalkData(talkData); @@ -181,7 +179,7 @@ class UdpTalkDataHandler extends ScpMessageBaseHandle // // 转pcm数据 // List pcmBytes = G711().convertList(g711Data); // talkData.content = pcmBytes; - talkDataRepository.addTalkData(talkData); + // talkDataRepository.addTalkData(talkData); } catch (e) { print('Error decoding G.711 to PCM: $e'); } diff --git a/lib/talk/starChart/handle/impl/udp_talk_request_handler.dart b/lib/talk/starChart/handle/impl/udp_talk_request_handler.dart index f6c90d0e..c55043bc 100644 --- a/lib/talk/starChart/handle/impl/udp_talk_request_handler.dart +++ b/lib/talk/starChart/handle/impl/udp_talk_request_handler.dart @@ -73,9 +73,19 @@ class UdpTalkRequestHandler extends ScpMessageBaseHandle // 启动对讲请求超时定时器 talkeRequestOverTimeTimerManager.start(); // 收到呼叫请求，跳转到接听页面 - Get.toNamed( - Routers.starChartTalkView, - ); + if (startChartManage + .getDefaultTalkExpect() + .videoType + .indexOf(VideoTypeE.H264) == + -1) { + Get.toNamed( + Routers.starChartTalkView, + ); + } else { + Get.toNamed( + Routers.h264WebView, + ); + } } // 收到来电请求时进行本地通知 diff --git a/lib/talk/starChart/handle/other/h264_frame_buffer.dart b/lib/talk/starChart/handle/other/h264_frame_buffer.dart new file mode 100644 index 00000000..5ace9001 --- /dev/null +++ b/lib/talk/starChart/handle/other/h264_frame_buffer.dart @@ -0,0 +1,22 @@ +import 'dart:typed_data'; + +import 'package:star_lock/talk/starChart/proto/talk_data_h264_frame.pb.dart'; + +class H264FrameBuffer { + List frames = []; + + void addFrame(TalkDataH264Frame frame) { + frames.add(frame); + } + + Uint8List getCompleteStream() { + final List completeStream = []; + for (final frame in frames) { + // 添加起始码（假设为 0x00 0x00 0x01） + completeStream.addAll([0x00, 0x00, 0x01]); + // 添加帧数据 + completeStream.addAll(frame.frameData); + } + return Uint8List.fromList(completeStream); + } +} diff --git a/lib/talk/starChart/handle/other/h264_frame_handler.dart b/lib/talk/starChart/handle/other/h264_frame_handler.dart new file mode 100644 index 00000000..5eaa27df --- /dev/null +++ b/lib/talk/starChart/handle/other/h264_frame_handler.dart @@ -0,0 +1,84 @@ +import 'package:star_lock/app_settings/app_settings.dart'; +import '../../proto/talk_data_h264_frame.pb.dart'; + +class H264FrameHandler { + final Map _frameBuffer = {}; + final void Function(List frameData) onCompleteFrame; + int _lastProcessedSeq = -1; + + H264FrameHandler({required this.onCompleteFrame}); + + void handleFrame(TalkDataH264Frame frame) { + // 存储帧 + _frameBuffer[frame.frameSeq] = frame; + + // 检查是否可以组装完整的 GOP (Group of Pictures) + _tryAssembleFrames(frame.frameSeq); + } + + void _tryAssembleFrames(int currentSeq) { + // 找到连续的帧序列 + final List sortedSeqs = _frameBuffer.keys.toList()..sort(); + final List framesToProcess = []; + + // 从当前帧开始向前找到最近的 I 帧或 P 帧 + int? startFrameSeq; + for (var seq in sortedSeqs.reversed) { + final frame = _frameBuffer[seq]; + if (frame?.frameType == TalkDataH264Frame_FrameTypeE.I) { + startFrameSeq = seq; + break; + } else if (frame?.frameType == TalkDataH264Frame_FrameTypeE.P) { + // 检查 P 帧是否有对应的 I 帧 + if (_frameBuffer.containsKey(frame?.frameSeqI)) { + startFrameSeq = seq; + break; + } else { + // 丢弃没有对应 I 帧的 P 帧 + _frameBuffer.remove(seq); + } + } + } + + if (startFrameSeq != null) { + // 收集从 I 帧或 P 帧开始的连续帧 + int expectedSeq = startFrameSeq; + for (var seq in sortedSeqs.where((s) => s >= startFrameSeq!)) { + if (seq != expectedSeq) break; + framesToProcess.add(seq); + expectedSeq++; + } + + if (framesToProcess.isNotEmpty) { + _processFrames(framesToProcess); + } + } else { + _clearOldFrames(currentSeq); + } + } + + void _clearOldFrames(int currentSeq) { + // 清理比当前帧序列旧的帧 + _frameBuffer.removeWhere((seq, frame) => seq < currentSeq - 200); // 调整阈值 + } + + void _processFrames(List frameSeqs) { + // 按顺序组装帧数据 + final List assembledData = []; + + for (var seq in frameSeqs) { + final frame = _frameBuffer[seq]!; + assembledData.addAll(frame.frameData); + + // 处理完后从缓冲区移除 + _frameBuffer.remove(seq); + } + + // 回调完整的帧数据 + onCompleteFrame(assembledData); + } + + void clear() { + _frameBuffer.clear(); + } +} diff --git a/lib/talk/starChart/handle/other/talk_data_repository.dart b/lib/talk/starChart/handle/other/talk_data_repository.dart index 90fc77df..864aa4aa 100644 --- a/lib/talk/starChart/handle/other/talk_data_repository.dart +++ b/lib/talk/starChart/handle/other/talk_data_repository.dart @@ -1,5 +1,4 @@ import 'dart:async'; - import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; class TalkDataRepository { @@ -27,6 +26,9 @@ class TalkDataRepository { bool _isListening = false; + // 用于存储数据的缓冲区 + final List _buffer = []; + // 提供一个方法来获取 Stream Stream get talkDataStream => _talkDataStreamController.stream.transform( @@ -41,14 +43,11 @@ class TalkDataRepository { }, ), ); - final List _buffer = []; // 用于存储数据的缓冲区 // 提供一个方法来添加 TalkData 到 Stream - void addTalkData(TalkData talkData) async { + void addTalkData(TalkData talkData) { if (_isListening) { - Future.microtask(() { - _talkDataStreamController.add(talkData); - }); + _talkDataStreamController.add(talkData); } } diff --git a/lib/talk/starChart/handle/scp_message_base_handle.dart b/lib/talk/starChart/handle/scp_message_base_handle.dart index 4cfc04c1..c1225f0e 100644 --- a/lib/talk/starChart/handle/scp_message_base_handle.dart +++ b/lib/talk/starChart/handle/scp_message_base_handle.dart @@ -14,6 +14,7 @@ import 'package:star_lock/talk/starChart/constant/payload_type_constant.dart'; import 'package:star_lock/talk/starChart/constant/udp_constant.dart'; import 'package:star_lock/talk/starChart/entity/scp_message.dart'; +import 'package:star_lock/talk/starChart/handle/other/h264_frame_handler.dart'; import 'package:star_lock/talk/starChart/handle/other/talk_data_repository.dart'; import 'package:star_lock/talk/starChart/handle/other/talke_data_over_time_timer_manager.dart'; @@ -52,6 +53,15 @@ class ScpMessageBaseHandle { final audioManager = AudioPlayerManager(); + // 处理出完整帧数据后的回调 + final H264FrameHandler frameHandler = + H264FrameHandler(onCompleteFrame: (frameData) { + // 处理完整的帧数据 + TalkDataRepository.instance.addTalkData( + TalkData(contentType: TalkData_ContentTypeE.H264, content: frameData), + ); + }); + // 回复成功消息 void replySuccessMessage(ScpMessage scpMessage) { startChartManage.sendGenericRespSuccessMessage( diff --git a/lib/talk/starChart/proto/talk_data_h264_frame.pb.dart b/lib/talk/starChart/proto/talk_data_h264_frame.pb.dart index 4a77a31a..e0239875 100644 --- a/lib/talk/starChart/proto/talk_data_h264_frame.pb.dart +++ b/lib/talk/starChart/proto/talk_data_h264_frame.pb.dart @@ -22,6 +22,7 @@ class TalkDataH264Frame extends $pb.GeneratedMessage { $core.int? frameSeq, TalkDataH264Frame_FrameTypeE? frameType, $core.List<$core.int>? frameData, + $core.int? frameSeqI, }) { final $result = create(); if (frameSeq != null) { @@ -33,6 +34,9 @@ class TalkDataH264Frame extends $pb.GeneratedMessage { if (frameData != null) { $result.frameData = frameData; } + if (frameSeqI != null) { + $result.frameSeqI = frameSeqI; + } return $result; } TalkDataH264Frame._() : super(); @@ -43,6 +47,7 @@ class TalkDataH264Frame extends $pb.GeneratedMessage { ..a<$core.int>(1, _omitFieldNames ? '' : 'FrameSeq', $pb.PbFieldType.OU3, protoName: 'FrameSeq') ..e(2, _omitFieldNames ? '' : 'FrameType', $pb.PbFieldType.OE, protoName: 'FrameType', defaultOrMaker: TalkDataH264Frame_FrameTypeE.NONE, valueOf: TalkDataH264Frame_FrameTypeE.valueOf, enumValues: TalkDataH264Frame_FrameTypeE.values) ..a<$core.List<$core.int>>(3, _omitFieldNames ? '' : 'FrameData', $pb.PbFieldType.OY, protoName: 'FrameData') + ..a<$core.int>(4, _omitFieldNames ? '' : 'FrameSeqI', $pb.PbFieldType.OU3, protoName: 'FrameSeqI') ..hasRequiredFields = false ; @@ -95,6 +100,16 @@ class TalkDataH264Frame extends $pb.GeneratedMessage { $core.bool hasFrameData() => $_has(2); @$pb.TagNumber(3) void clearFrameData() => clearField(3); + + /// 帧序号I + @$pb.TagNumber(4) + $core.int get frameSeqI => $_getIZ(3); + @$pb.TagNumber(4) + set frameSeqI($core.int v) { $_setUnsignedInt32(3, v); } + @$pb.TagNumber(4) + $core.bool hasFrameSeqI() => $_has(3); + @$pb.TagNumber(4) + void clearFrameSeqI() => clearField(4); } diff --git a/lib/talk/starChart/proto/talk_data_h264_frame.pbjson.dart b/lib/talk/starChart/proto/talk_data_h264_frame.pbjson.dart index bc7b066c..6bf9aa9b 100644 --- a/lib/talk/starChart/proto/talk_data_h264_frame.pbjson.dart +++ b/lib/talk/starChart/proto/talk_data_h264_frame.pbjson.dart @@ -20,6 +20,7 @@ const TalkDataH264Frame$json = { {'1': 'FrameSeq', '3': 1, '4': 1, '5': 13, '10': 'FrameSeq'}, {'1': 'FrameType', '3': 2, '4': 1, '5': 14, '6': '.main.TalkDataH264Frame.FrameTypeE', '10': 'FrameType'}, {'1': 'FrameData', '3': 3, '4': 1, '5': 12, '10': 'FrameData'}, + {'1': 'FrameSeqI', '3': 4, '4': 1, '5': 13, '10': 'FrameSeqI'}, ], '4': [TalkDataH264Frame_FrameTypeE$json], }; @@ -38,6 +39,6 @@ const TalkDataH264Frame_FrameTypeE$json = { final $typed_data.Uint8List talkDataH264FrameDescriptor = $convert.base64Decode( 'ChFUYWxrRGF0YUgyNjRGcmFtZRIaCghGcmFtZVNlcRgBIAEoDVIIRnJhbWVTZXESQAoJRnJhbW' 'VUeXBlGAIgASgOMiIubWFpbi5UYWxrRGF0YUgyNjRGcmFtZS5GcmFtZVR5cGVFUglGcmFtZVR5' - 'cGUSHAoJRnJhbWVEYXRhGAMgASgMUglGcmFtZURhdGEiJAoKRnJhbWVUeXBlRRIICgROT05FEA' - 'ASBQoBSRABEgUKAVAQAg=='); + 'cGUSHAoJRnJhbWVEYXRhGAMgASgMUglGcmFtZURhdGESHAoJRnJhbWVTZXFJGAQgASgNUglGcm' + 'FtZVNlcUkiJAoKRnJhbWVUeXBlRRIICgROT05FEAASBQoBSRABEgUKAVAQAg=='); diff --git a/lib/talk/starChart/proto/talk_data_h264_frame.proto b/lib/talk/starChart/proto/talk_data_h264_frame.proto index 09815d16..1d08adef 100644 --- a/lib/talk/starChart/proto/talk_data_h264_frame.proto +++ b/lib/talk/starChart/proto/talk_data_h264_frame.proto @@ -15,4 +15,6 @@ message TalkDataH264Frame { FrameTypeE FrameType = 2; // 帧数据 bytes FrameData = 3; + // 帧序号I + uint32 FrameSeqI = 4; } diff --git a/lib/talk/starChart/star_chart_manage.dart b/lib/talk/starChart/star_chart_manage.dart index 1a3d2a85..91617794 100644 --- a/lib/talk/starChart/star_chart_manage.dart +++ b/lib/talk/starChart/star_chart_manage.dart @@ -112,7 +112,7 @@ class StartChartManage { // 默认通话的期望数据格式 TalkExpectReq _defaultTalkExpect = TalkExpectReq( - videoType: [VideoTypeE.IMAGE], + videoType: [VideoTypeE.H264], audioType: [AudioTypeE.G711], ); @@ -419,9 +419,15 @@ class StartChartManage { if (talkStatus.status != TalkStatus.proactivelyCallWaitingAnswer) { // 停止播放铃声 // AudioPlayerManager().playRingtone(); - Get.toNamed( - Routers.starChartTalkView, - ); + if (_defaultTalkExpect.videoType.contains(VideoTypeE.H264)) { + Get.toNamed( + Routers.h264WebView, + ); + } else { + Get.toNamed( + Routers.starChartTalkView, + ); + } } talkRequestTimer ??= Timer.periodic( Duration( @@ -1113,15 +1119,19 @@ class StartChartManage { void reSetDefaultTalkExpect() { _defaultTalkExpect = TalkExpectReq( - videoType: [VideoTypeE.IMAGE], + videoType: [VideoTypeE.H264], audioType: [AudioTypeE.G711], ); } + TalkExpectReq getDefaultTalkExpect() { + return _defaultTalkExpect; + } + /// 修改预期接收到的数据 void sendOnlyImageVideoTalkExpectData() { final talkExpectReq = TalkExpectReq( - videoType: [VideoTypeE.IMAGE], + videoType: [VideoTypeE.H264], audioType: [], ); changeTalkExpectDataTypeAndReStartTalkExpectMessageTimer( @@ -1131,7 +1141,7 @@ class StartChartManage { /// 修改预期接收到的数据 void sendImageVideoAndG711AudioTalkExpectData() { final talkExpectReq = TalkExpectReq( - videoType: [VideoTypeE.IMAGE], + videoType: [VideoTypeE.H264], audioType: [AudioTypeE.G711], ); changeTalkExpectDataTypeAndReStartTalkExpectMessageTimer( diff --git a/lib/talk/starChart/webView/h264_web_logic.dart b/lib/talk/starChart/webView/h264_web_logic.dart new file mode 100644 index 00000000..e8e28784 --- /dev/null +++ b/lib/talk/starChart/webView/h264_web_logic.dart @@ -0,0 +1,434 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:ui' as ui; +import 'dart:math'; // Import the math package to use sqrt + +import 'package:flutter/foundation.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_pcm_sound/flutter_pcm_sound.dart'; +import 'package:flutter_screen_recording/flutter_screen_recording.dart'; +import 'package:flutter_voice_processor/flutter_voice_processor.dart'; +import 'package:gallery_saver/gallery_saver.dart'; +import 'package:get/get.dart'; +import 'package:image_gallery_saver/image_gallery_saver.dart'; +import 'package:path_provider/path_provider.dart'; +import 'package:permission_handler/permission_handler.dart'; +import 'package:star_lock/app_settings/app_settings.dart'; +import 'package:star_lock/login/login/entity/LoginEntity.dart'; +import 'package:star_lock/main/lockDetail/lockDetail/lockDetail_logic.dart'; +import 'package:star_lock/main/lockDetail/lockDetail/lockDetail_state.dart'; +import 'package:star_lock/main/lockDetail/lockDetail/lockNetToken_entity.dart'; +import 'package:star_lock/main/lockDetail/lockSet/lockSet/lockSetInfo_entity.dart'; +import 'package:star_lock/main/lockMian/entity/lockListInfo_entity.dart'; +import 'package:star_lock/network/api_repository.dart'; +import 'package:star_lock/talk/call/g711.dart'; +import 'package:star_lock/talk/starChart/constant/talk_status.dart'; +import 'package:star_lock/talk/starChart/proto/talk_data.pb.dart'; +import 'package:star_lock/talk/starChart/proto/talk_expect.pb.dart'; +import 'package:star_lock/talk/starChart/star_chart_manage.dart'; +import 'package:star_lock/talk/starChart/views/talkView/talk_view_state.dart'; +import 'package:star_lock/talk/starChart/webView/h264_web_view_state.dart'; +import 'package:star_lock/tools/bugly/bugly_tool.dart'; +import 'package:webview_flutter/webview_flutter.dart'; + +import '../../../../tools/baseGetXController.dart'; + +class H264WebViewLogic extends BaseGetXController { + final H264WebViewState state = H264WebViewState(); + + final LockDetailState lockDetailState = Get.put(LockDetailLogic()).state; + + @override + void onInit() { + super.onInit(); + // 初始化 WebView 控制器 + state.webViewController = WebViewController() + ..setJavaScriptMode(JavaScriptMode.unrestricted) + ..enableZoom(false) + ..addJavaScriptChannel( + 'Flutter', + onMessageReceived: (message) { + print("来自 HTML 的消息: ${message.message}"); + }, + ); + + state.isShowLoading.value = true; + // 加载本地 HTML + _loadLocalHtml(); + // 创建流数据监听 + _createFramesStreamListen(); + + _startListenTalkStatus(); + state.talkStatus.value = state.startChartTalkStatus.status; + // 初始化音频播放器 + _initFlutterPcmSound(); + // 初始化录音控制器 + _initAudioRecorder(); + } + + /// 初始化音频录制器 + void _initAudioRecorder() { + state.voiceProcessor = VoiceProcessor.instance; + } + + /// 初始化音频播放器 + void _initFlutterPcmSound() { + const int sampleRate = 8000; + FlutterPcmSound.setLogLevel(LogLevel.none); + FlutterPcmSound.setup(sampleRate: sampleRate, channelCount: 1); + // 设置 feed 阈值 + if (Platform.isAndroid) { + FlutterPcmSound.setFeedThreshold(1024); // Android 平台的特殊处理 + } else { + FlutterPcmSound.setFeedThreshold(2000); // 非 Android 平台的处理 + } + } + + void _createFramesStreamListen() async { + state.talkDataRepository.talkDataStream.listen((TalkData event) async { + // 发送数据给js处理 + _sendBufferedData(event.content); + }); + } + + /// 加载html文件 + Future _loadLocalHtml() async { + // 加载 HTML 文件内容 + final String fileHtmlContent = + await rootBundle.loadString('assets/html/h264.html'); + + // 加载 JS 文件内容 + final String jsContent = + await rootBundle.loadString('assets/html/jmuxer.min.js'); + + // 将 JS 文件内容嵌入到 HTML 中 + final String htmlWithJs = fileHtmlContent.replaceAll( + '', // 替换掉引用外部 JS 的标签 + '' // 使用内联方式嵌入 JS 内容 + ); + + // 加载最终的 HTML 字符串到 WebView 中 + if (state.webViewController != null) { + state.webViewController.loadHtmlString(htmlWithJs); // 设置 baseUrl 避免资源加载问题 + } + } + + // 修改后的发送方法 + _sendBufferedData(List buffer) async { + // 原始发送逻辑 + String jsCode = "feedDataFromFlutter($buffer);"; + await state.webViewController.runJavaScript(jsCode); + + if (state.isShowLoading.isTrue) { + await Future.delayed(Duration(seconds: 1)); + state.isShowLoading.value = false; + } + } + + /// 监听对讲状态 + void _startListenTalkStatus() { + state.startChartTalkStatus.statusStream.listen((talkStatus) { + state.talkStatus.value = talkStatus; + switch (talkStatus) { + case TalkStatus.rejected: + case TalkStatus.hangingUpDuring: + case TalkStatus.notTalkData: + case TalkStatus.notTalkPing: + case TalkStatus.end: + _handleInvalidTalkStatus(); + break; + case TalkStatus.answeredSuccessfully: + state.oneMinuteTimeTimer?.cancel(); // 取消旧定时器 + state.oneMinuteTimeTimer ??= + Timer.periodic(const Duration(seconds: 1), (Timer t) { + if (state.isShowLoading.isFalse) { + state.oneMinuteTime.value++; + if (state.oneMinuteTime.value >= 60) { + t.cancel(); // 取消定时器 + state.oneMinuteTime.value = 0; + } + } + }); + break; + default: + // 其他状态的处理 + break; + } + }); + } + + /// 更新发送预期数据 + void updateTalkExpect() { + TalkExpectReq talkExpectReq = TalkExpectReq(); + state.isOpenVoice.value = !state.isOpenVoice.value; + if (!state.isOpenVoice.value) { + talkExpectReq = TalkExpectReq( + videoType: [VideoTypeE.IMAGE], + audioType: [], + ); + showToast('已静音'.tr); + } else { + talkExpectReq = TalkExpectReq( + videoType: [VideoTypeE.IMAGE], + audioType: [AudioTypeE.G711], + ); + } + + /// 修改发送预期数据 + StartChartManage().changeTalkExpectDataTypeAndReStartTalkExpectMessageTimer( + talkExpect: talkExpectReq); + } + + /// 处理无效通话状态 + void _handleInvalidTalkStatus() {} + + /// 截图并保存到相册 + Future captureAndSavePng() async { + try { + if (state.globalKey.currentContext == null) { + AppLog.log('截图失败: 未找到当前上下文'); + return; + } + final RenderRepaintBoundary boundary = state.globalKey.currentContext! + .findRenderObject()! as RenderRepaintBoundary; + final ui.Image image = await boundary.toImage(); + final ByteData? byteData = + await image.toByteData(format: ui.ImageByteFormat.png); + + if (byteData == null) { + AppLog.log('截图失败: 图像数据为空'); + return; + } + final Uint8List pngBytes = byteData.buffer.asUint8List(); + + // 获取应用程序的文档目录 + final Directory directory = await getApplicationDocumentsDirectory(); + final String imagePath = '${directory.path}/screenshot.png'; + + // 将截图保存为文件 + final File imgFile = File(imagePath); + await imgFile.writeAsBytes(pngBytes); + + // 将截图保存到相册 + await ImageGallerySaver.saveFile(imagePath); + + AppLog.log('截图保存路径: $imagePath'); + showToast('截图已保存到相册'.tr); + } catch (e) { + AppLog.log('截图失败: $e'); + } + } + + // 发起接听命令 + void initiateAnswerCommand() { + StartChartManage().startTalkAcceptTimer(); + } + + //开始录音 + Future startProcessingAudio() async { + // 增加录音帧监听器和错误监听器 + state.voiceProcessor?.addFrameListener(_onFrame); + state.voiceProcessor?.addErrorListener(_onError); + try { + if (await state.voiceProcessor?.hasRecordAudioPermission() ?? false) { + await state.voiceProcessor?.start(state.frameLength, state.sampleRate); + final bool? isRecording = await state.voiceProcessor?.isRecording(); + state.isRecordingAudio.value = isRecording!; + state.startRecordingAudioTime.value = DateTime.now(); + } else { + // state.errorMessage.value = 'Recording permission not granted'; + } + } on PlatformException catch (ex) { + // state.errorMessage.value = 'Failed to start recorder: $ex'; + } + state.isOpenVoice.value = false; + } + + /// 停止录音 + Future stopProcessingAudio() async { + try { + await state.voiceProcessor?.stop(); + state.voiceProcessor?.removeFrameListener(_onFrame); + state.udpSendDataFrameNumber = 0; + // 记录结束时间 + state.endRecordingAudioTime.value = DateTime.now(); + + // 计算录音的持续时间 + final duration = state.endRecordingAudioTime.value! + .difference(state.startRecordingAudioTime.value!); + + state.recordingAudioTime.value = duration.inSeconds; + } on PlatformException catch (ex) { + // state.errorMessage.value = 'Failed to stop recorder: $ex'; + } finally { + final bool? isRecording = await state.voiceProcessor?.isRecording(); + state.isRecordingAudio.value = isRecording!; + state.isOpenVoice.value = true; + } + } + + // 音频帧处理 + Future _onFrame(List frame) async { + // 预处理和转码操作放到异步计算线程 + // final processedFrame = await compute(preprocessAudio, frame); + // final list = listLinearToALaw(processedFrame); + final List processedFrame = preprocessAudio(frame); + final List list = listLinearToALaw(processedFrame); + + final int ms = DateTime.now().millisecondsSinceEpoch - + state.startRecordingAudioTime.value.millisecondsSinceEpoch; + + // 发送音频数据到UDP + await StartChartManage().sendTalkDataMessage( + talkData: TalkData( + content: list, + contentType: TalkData_ContentTypeE.G711, + durationMs: ms, + ), + ); + } + + /// 挂断 + void udpHangUpAction() async { + if (state.talkStatus.value == TalkStatus.answeredSuccessfully) { + // 如果是通话中就挂断 + StartChartManage().startTalkHangupMessageTimer(); + } else { + // 拒绝 + StartChartManage().startTalkRejectMessageTimer(); + } + Get.back(); + } + + // 远程开锁 + Future remoteOpenLock() async { + final lockPeerId = StartChartManage().lockPeerId; + final lockListPeerId = StartChartManage().lockListPeerId; + int lockId = lockDetailState.keyInfos.value.lockId ?? 0; + + // 如果锁列表获取到peerId，代表有多个锁，使用锁列表的peerId + // 从列表中遍历出对应的peerId + lockListPeerId.forEach((element) { + if (element.network?.peerId == lockPeerId) { + lockId = element.lockId ?? 0; + } + }); + + final LockSetInfoEntity lockSetInfoEntity = + await ApiRepository.to.getLockSettingInfoData( + lockId: lockId.toString(), + ); + if (lockSetInfoEntity.errorCode!.codeIsSuccessful) { + if (lockSetInfoEntity.data?.lockFeature?.remoteUnlock == 1 && + lockSetInfoEntity.data?.lockSettingInfo?.remoteUnlock == 1) { + final LoginEntity entity = await ApiRepository.to + .remoteOpenLock(lockId: lockId.toString(), timeOut: 60); + if (entity.errorCode!.codeIsSuccessful) { + showToast('已开锁'.tr); + StartChartManage().lockListPeerId = []; + } + } else { + showToast('该锁的远程开锁功能未启用'.tr); + } + } + } + + List preprocessAudio(List pcmList) { + // 简单的降噪处理 + final List processedList = []; + for (int pcmVal in pcmList) { + // 简单的降噪示例：将小于阈值的信号置为0 + if (pcmVal.abs() < 200) { + pcmVal = 0; + } + processedList.add(pcmVal); + } + return processedList; + } + + List listLinearToALaw(List pcmList) { + final List aLawList = []; + for (int pcmVal in pcmList) { + final int aLawVal = linearToALaw(pcmVal); + aLawList.add(aLawVal); + } + return aLawList; + } + + int linearToALaw(int pcmVal) { + const int ALAW_MAX = 0x7FFF; // 32767 + const int ALAW_BIAS = 0x84; // 132 + + int mask; + int seg; + int aLawVal; + + // Handle sign + if (pcmVal < 0) { + pcmVal = -pcmVal; + mask = 0x7F; // 127 (sign bit is 1) + } else { + mask = 0xFF; // 255 (sign bit is 0) + } + + // Add bias and clamp to ALAW_MAX + pcmVal += ALAW_BIAS; + if (pcmVal > ALAW_MAX) { + pcmVal = ALAW_MAX; + } + + // Determine segment + seg = search(pcmVal); + + // Calculate A-law value + if (seg >= 8) { + aLawVal = 0x7F ^ mask; // Clamp to maximum value + } else { + int quantized = (pcmVal >> (seg + 3)) & 0xF; + aLawVal = (seg << 4) | quantized; + aLawVal ^= 0xD5; // XOR with 0xD5 to match standard A-law table + } + + return aLawVal; + } + + int search(int val) { + final List table = [ + 0xFF, // Segment 0 + 0x1FF, // Segment 1 + 0x3FF, // Segment 2 + 0x7FF, // Segment 3 + 0xFFF, // Segment 4 + 0x1FFF, // Segment 5 + 0x3FFF, // Segment 6 + 0x7FFF // Segment 7 + ]; + const int size = 8; + for (int i = 0; i < size; i++) { + if (val <= table[i]) { + return i; + } + } + return size; + } + +// 错误监听 + void _onError(VoiceProcessorException error) { + AppLog.log(error.message!); + } + + @override + void dispose() { + // TODO: implement dispose + super.dispose(); + StartChartManage().startTalkHangupMessageTimer(); + state.animationController.dispose(); + state.webViewController.clearCache(); + state.webViewController.reload(); + state.oneMinuteTimeTimer?.cancel(); + state.oneMinuteTimeTimer = null; + stopProcessingAudio(); + StartChartManage().reSetDefaultTalkExpect(); + } +} diff --git a/lib/talk/starChart/webView/h264_web_view.dart b/lib/talk/starChart/webView/h264_web_view.dart index 2392c5fa..51aa30d8 100644 --- a/lib/talk/starChart/webView/h264_web_view.dart +++ b/lib/talk/starChart/webView/h264_web_view.dart @@ -1,11 +1,18 @@ import 'dart:async'; import 'dart:convert'; +import 'dart:math'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart' show ByteData, Uint8List, rootBundle; import 'package:flutter_screenutil/flutter_screenutil.dart'; +import 'package:get/get.dart'; import 'package:star_lock/app_settings/app_colors.dart'; +import 'package:star_lock/app_settings/app_settings.dart'; +import 'package:star_lock/talk/starChart/constant/talk_status.dart'; import 'package:star_lock/talk/starChart/handle/other/talk_data_repository.dart'; import 'package:star_lock/talk/starChart/proto/talk_data.pbserver.dart'; +import 'package:star_lock/talk/starChart/star_chart_manage.dart'; +import 'package:star_lock/talk/starChart/webView/h264_web_logic.dart'; +import 'package:star_lock/talk/starChart/webView/h264_web_view_state.dart'; import 'package:star_lock/tools/titleAppBar.dart'; import 'package:webview_flutter/webview_flutter.dart'; @@ -14,169 +21,400 @@ class H264WebView extends StatefulWidget { _H264WebViewState createState() => _H264WebViewState(); } -class _H264WebViewState extends State { - late final WebViewController _controller; - Timer? timer; - Timer? _sendTimer; - - // 私有缓冲区，外部无法直接访问 - final List _buffer = []; - - // 发送数据至html文件间隔时间 - final int sendDataToHtmlIntervalTime = 820; - - // 通话数据流的单例流数据处理类 - final TalkDataRepository talkDataRepository = TalkDataRepository.instance; +class _H264WebViewState extends State + with TickerProviderStateMixin { + final H264WebViewLogic logic = Get.put(H264WebViewLogic()); + final H264WebViewState state = Get.find().state; @override void initState() { + // TODO: implement initState super.initState(); + state.animationController = AnimationController( + vsync: this, // 确保使用的TickerProvider是当前Widget + duration: const Duration(seconds: 1), + ); - _controller = WebViewController() - ..setJavaScriptMode(JavaScriptMode.unrestricted) - ..enableZoom(false) - ..addJavaScriptChannel( - 'Flutter', - onMessageReceived: (message) { - print("来自 HTML 的消息: ${message.message}"); - }, - ); - - // 加载本地 HTML - _loadLocalHtml(); - simulateStreamFromAsset(); - _sendFramesToHtml(); - } - - void simulateStreamFromAsset() async { - // 读取 assets 文件 - final ByteData data = await rootBundle.load('assets/talk.h264'); - final List byteData = data.buffer.asUint8List(); - int current = 0; - int start = 0; - int end = 0; - final List chunks = extractChunks(byteData); - // 定时器控制发送数据块的节奏 - timer ??= Timer.periodic(Duration(milliseconds: 10), (timer) { - if (current >= chunks.length) { - print('数据已经发完,重新进行发送'); - start = 0; - end = 0; - current = 0; - timer.cancel(); - return; - } - // 提取 NALU 边界并生成 chunks - end = chunks[current]; - current++; - List frameData = byteData.sublist(start, end); - if (frameData.length == 0) timer.cancel(); - - talkDataRepository.addTalkData(TalkData(contentType: TalkData_ContentTypeE.H264,content: frameData)); - start = end; - }); - } - - void _sendFramesToHtml() async { - // 接收到流数据，保存到缓冲区 - // talkDataRepository.talkDataStream.listen((TalkData event) async { - // _buffer.addAll(event.content); - // }); - // 缓冲800ms的数据，定时发送 - _sendTimer ??= Timer.periodic( - Duration(milliseconds: sendDataToHtmlIntervalTime), (timer) async { - // 发送累积的数据 - if (_buffer.isNotEmpty) { - await _sendBufferedData(_buffer); - _buffer.clear(); // 清空缓冲区 + state.animationController.repeat(); + //动画开始、结束、向前移动或向后移动时会调用StatusListener + state.animationController.addStatusListener((AnimationStatus status) { + if (status == AnimationStatus.completed) { + state.animationController.reset(); + state.animationController.forward(); + } else if (status == AnimationStatus.dismissed) { + state.animationController.reset(); + state.animationController.forward(); } }); } - // 提取 NALU 边界并生成 chunks - List extractChunks(List byteData) { - int i = 0; - int length = byteData.length; - int naluCount = 0; - int value; - int state = 0; - int lastIndex = 0; - List result = []; - const minNaluPerChunk = 22; // 每个数据块包含的最小NALU数量 - - while (i < length) { - value = byteData[i++]; - // finding 3 or 4-byte start codes (00 00 01 OR 00 00 00 01) - switch (state) { - case 0: - if (value == 0) { - state = 1; - } - break; - case 1: - if (value == 0) { - state = 2; - } else { - state = 0; - } - break; - case 2: - case 3: - if (value == 0) { - state = 3; - } else if (value == 1 && i < length) { - if (lastIndex > 0) { - naluCount++; - } - if (naluCount >= minNaluPerChunk) { - result.add(lastIndex - state - 1); - naluCount = 0; - } - state = 0; - lastIndex = i; - } else { - state = 0; - } - break; - default: - break; - } - } - - if (naluCount > 0) { - result.add(lastIndex); - } - - - - return result; - } - - /// 加载html文件 - Future _loadLocalHtml() async { - final String fileHtmlContent = - await rootBundle.loadString('assets/html/h264.html'); - _controller.loadHtmlString(fileHtmlContent); - } - - // 发送数据给js处理 - _sendBufferedData(List buffer) async { - String jsCode = "feedDataFromFlutter(${buffer});"; - await _controller.runJavaScript(jsCode); - } - @override Widget build(BuildContext context) { - return WebViewWidget(controller: _controller); + return WillPopScope( + onWillPop: () async { + // 返回 false 表示禁止退出 + return false; + }, + child: SizedBox( + width: 1.sw, + height: 1.sh, + child: Stack( + alignment: Alignment.center, + children: [ + Obx(() { + final double screenWidth = MediaQuery.of(context).size.width; + final double screenHeight = MediaQuery.of(context).size.height; + return state.isShowLoading.value + ? Image.asset( + 'images/main/monitorBg.png', + width: screenWidth, + height: screenHeight, + fit: BoxFit.cover, + ) + : WebViewWidget( + controller: state.webViewController, + ); + }), + Obx( + () => state.isShowLoading.value + ? Positioned( + bottom: 310.h, + child: Text( + '正在创建安全连接...'.tr, + style: TextStyle(color: Colors.black, fontSize: 26.sp), + ), + ) + : Container(), + ), + Obx( + () => state.isShowLoading.isFalse + ? Positioned( + top: ScreenUtil().statusBarHeight + 75.h, + width: 1.sw, + child: Obx( + () { + final String sec = (state.oneMinuteTime.value % 60) + .toString() + .padLeft(2, '0'); + final String min = (state.oneMinuteTime.value ~/ 60) + .toString() + .padLeft(2, '0'); + return Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Text( + '$min:$sec', + style: TextStyle( + fontSize: 26.sp, color: Colors.white), + ), + ], + ); + }, + ), + ) + : Container(), + ), + Positioned( + bottom: 10.w, + child: Container( + width: 1.sw - 30.w * 2, + // height: 300.h, + margin: EdgeInsets.all(30.w), + decoration: BoxDecoration( + color: Colors.black.withOpacity(0.2), + borderRadius: BorderRadius.circular(20.h)), + child: Column( + children: [ + SizedBox(height: 20.h), + bottomTopBtnWidget(), + SizedBox(height: 20.h), + bottomBottomBtnWidget(), + SizedBox(height: 20.h), + ], + ), + ), + ), + Obx(() => state.isShowLoading.isTrue + ? buildRotationTransition() + : Container()), + Obx(() => state.isLongPressing.value + ? Positioned( + top: 80.h, + left: 0, + right: 0, + child: Center( + child: Container( + padding: EdgeInsets.all(10.w), + decoration: BoxDecoration( + color: Colors.black.withOpacity(0.7), + borderRadius: BorderRadius.circular(10.w), + ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(Icons.mic, color: Colors.white, size: 24.w), + SizedBox(width: 10.w), + Text( + '正在说话...'.tr, + style: TextStyle( + fontSize: 20.sp, color: Colors.white), + ), + ], + ), + ), + ), + ) + : Container()), + ], + ), + ), + ); } + Widget bottomTopBtnWidget() { + return Row(mainAxisAlignment: MainAxisAlignment.center, children: [ + // 打开关闭声音 + GestureDetector( + onTap: () { + if (state.talkStatus.value == TalkStatus.answeredSuccessfully) { + // 打开关闭声音 + logic.updateTalkExpect(); + } + }, + child: Container( + width: 50.w, + height: 50.w, + padding: EdgeInsets.all(5.w), + child: Obx(() => Image( + width: 40.w, + height: 40.w, + image: state.isOpenVoice.value + ? const AssetImage( + 'images/main/icon_lockDetail_monitoringOpenVoice.png') + : const AssetImage( + 'images/main/icon_lockDetail_monitoringCloseVoice.png'))), + ), + ), + SizedBox(width: 50.w), + // 截图 + GestureDetector( + onTap: () async { + if (state.talkStatus.value == TalkStatus.answeredSuccessfully) { + await logic.captureAndSavePng(); + } + }, + child: Container( + width: 50.w, + height: 50.w, + padding: EdgeInsets.all(5.w), + child: Image( + width: 40.w, + height: 40.w, + image: const AssetImage( + 'images/main/icon_lockDetail_monitoringScreenshot.png')), + ), + ), + SizedBox(width: 50.w), + // 录制 + GestureDetector( + onTap: () async { + logic.showToast('功能暂未开放'.tr); + // if ( + // state.talkStatus.value == TalkStatus.answeredSuccessfully) { + // if (state.isRecordingScreen.value) { + // await logic.stopRecording(); + // } else { + // await logic.startRecording(); + // } + // } + }, + child: Container( + width: 50.w, + height: 50.w, + padding: EdgeInsets.all(5.w), + child: Image( + width: 40.w, + height: 40.w, + fit: BoxFit.fill, + image: const AssetImage( + 'images/main/icon_lockDetail_monitoringScreenRecording.png'), + ), + ), + ), + SizedBox(width: 50.w), + GestureDetector( + onTap: () { + logic.showToast('功能暂未开放'.tr); + }, + child: Image( + width: 28.w, + height: 28.w, + fit: BoxFit.fill, + image: const AssetImage('images/main/icon_lockDetail_rectangle.png'), + ), + ), + ]); + } + + Widget bottomBottomBtnWidget() { + return Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + // 接听 + Obx( + () => bottomBtnItemWidget( + getAnswerBtnImg(), + getAnswerBtnName(), + Colors.white, + longPress: () async { + if (state.talkStatus.value == TalkStatus.answeredSuccessfully) { + // 启动录音 + logic.startProcessingAudio(); + state.isLongPressing.value = true; + } + }, + longPressUp: () async { + // 停止录音 + logic.stopProcessingAudio(); + state.isLongPressing.value = false; + }, + onClick: () async { + if (state.talkStatus.value == + TalkStatus.passiveCallWaitingAnswer) { + // 接听 + logic.initiateAnswerCommand(); + } + }, + ), + ), + bottomBtnItemWidget( + 'images/main/icon_lockDetail_hangUp.png', '挂断'.tr, Colors.red, + onClick: () { + // 挂断 + logic.udpHangUpAction(); + }), + bottomBtnItemWidget( + 'images/main/icon_lockDetail_monitoringUnlock.png', + '开锁'.tr, + AppColors.mainColor, + onClick: () { + // if (state.talkStatus.value == TalkStatus.answeredSuccessfully && + // state.listData.value.length > 0) { + // logic.udpOpenDoorAction(); + logic.remoteOpenLock(); + // } + // if (UDPManage().remoteUnlock == 1) { + // logic.udpOpenDoorAction(); + // showDeletPasswordAlertDialog(context); + // } else { + // logic.showToast('请在锁设置中开启远程开锁'.tr); + // } + }, + ) + ]); + } + + String getAnswerBtnImg() { + switch (state.talkStatus.value) { + case TalkStatus.passiveCallWaitingAnswer: + return 'images/main/icon_lockDetail_monitoringAnswerCalls.png'; + case TalkStatus.answeredSuccessfully: + case TalkStatus.proactivelyCallWaitingAnswer: + return 'images/main/icon_lockDetail_monitoringUnTalkback.png'; + default: + return 'images/main/icon_lockDetail_monitoringAnswerCalls.png'; + } + } + + String getAnswerBtnName() { + switch (state.talkStatus.value) { + case TalkStatus.passiveCallWaitingAnswer: + return '接听'.tr; + case TalkStatus.proactivelyCallWaitingAnswer: + case TalkStatus.answeredSuccessfully: + return '长按说话'.tr; + default: + return '接听'.tr; + } + } + + Widget bottomBtnItemWidget( + String iconUrl, + String name, + Color backgroundColor, { + required Function() onClick, + Function()? longPress, + Function()? longPressUp, + }) { + double wh = 80.w; + return GestureDetector( + onTap: onClick, + onLongPress: longPress, + onLongPressUp: longPressUp, + child: SizedBox( + height: 160.w, + width: 140.w, + child: Column( + crossAxisAlignment: CrossAxisAlignment.center, + children: [ + Container( + width: wh, + height: wh, + constraints: BoxConstraints( + minWidth: wh, + ), + decoration: BoxDecoration( + color: backgroundColor, + borderRadius: BorderRadius.circular((wh + 10.w * 2) / 2), + ), + padding: EdgeInsets.all(20.w), + child: Image.asset(iconUrl, fit: BoxFit.fitWidth), + ), + SizedBox(height: 20.w), + Text( + name, + style: TextStyle(fontSize: 20.sp, color: Colors.white), + textAlign: TextAlign.center, // 当文本超出指定行数时，使用省略号表示 + maxLines: 2, // 设置最大行数为1 + ) + ], + ), + ), + ); + } + + //旋转动画 + Widget buildRotationTransition() { + return Positioned( + left: ScreenUtil().screenWidth / 2 - 220.w / 2, + top: ScreenUtil().screenHeight / 2 - 220.w / 2 - 150.h, + child: GestureDetector( + child: RotationTransition( + //设置动画的旋转中心 + alignment: Alignment.center, + //动画控制器 + turns: state.animationController, + //将要执行动画的子view + child: AnimatedOpacity( + opacity: 0.5, + duration: const Duration(seconds: 2), + child: Image.asset( + 'images/main/realTime_connecting.png', + width: 220.w, + height: 220.w, + ), + ), + ), + onTap: () { + state.animationController.forward(); + }, + ), + ); + } @override void dispose() { - timer?.cancel(); - timer = null; - _sendTimer?.cancel(); - timer = null; - // talkDataRepository.dispose(); + state.animationController.dispose(); // 确保释放控制器 super.dispose(); + } } diff --git a/lib/talk/starChart/webView/h264_web_view_state.dart b/lib/talk/starChart/webView/h264_web_view_state.dart new file mode 100644 index 00000000..2ae11041 --- /dev/null +++ b/lib/talk/starChart/webView/h264_web_view_state.dart @@ -0,0 +1,52 @@ +import 'dart:async'; + +import 'package:flutter/cupertino.dart'; +import 'package:flutter_voice_processor/flutter_voice_processor.dart'; +import 'package:get/get.dart'; +import 'package:star_lock/talk/starChart/constant/talk_status.dart'; +import 'package:star_lock/talk/starChart/handle/other/talk_data_repository.dart'; +import 'package:star_lock/talk/starChart/status/star_chart_talk_status.dart'; +import 'package:star_lock/talk/starChart/views/talkView/talk_view_state.dart'; +import 'package:webview_flutter/webview_flutter.dart'; + +class H264WebViewState { + GlobalKey globalKey = GlobalKey(); + int udpSendDataFrameNumber = 0; // 帧序号 + late AnimationController animationController; + + // webview 控制器 + late final WebViewController webViewController; + + // 获取 startChartTalkStatus 的唯一实例 + final StartChartTalkStatus startChartTalkStatus = + StartChartTalkStatus.instance; + Rx talkStatus = TalkStatus.none.obs; //星图对讲状态 + + RxBool isShowLoading = true.obs; + + Timer? oneMinuteTimeTimer; // 定时器超过60秒关闭当前界面 + RxInt oneMinuteTime = 0.obs; // 定时器秒数 + + RxBool isLongPressing = false.obs; // 是否长按说话 + final TalkDataRepository talkDataRepository = TalkDataRepository.instance; + RxInt lastFrameTimestamp = 0.obs; // 上一帧的时间戳,用来判断网络环境 + Rx networkStatus = + NetworkStatus.normal.obs; // 网络状态：0-正常 1-网络卡顿 2-网络延迟 3-网络丢包 + RxInt alertCount = 0.obs; // 网络状态提示计数器 + RxInt maxAlertNumber = 3.obs; // 网络状态提示最大提示次数 + RxBool isOpenVoice = true.obs; // 是否打开声音 + RxBool isRecordingScreen = false.obs; // 是否录屏中 + RxBool isRecordingAudio = false.obs; // 是否录音中 + Rx startRecordingAudioTime = DateTime.now().obs; // 开始录音时间 + Rx endRecordingAudioTime = DateTime.now().obs; // 结束录音时间 + RxInt recordingAudioTime = 0.obs; // 录音时间持续时间 + RxDouble fps = 0.0.obs; // 添加 FPS 计数 + late VoiceProcessor? voiceProcessor; // 音频处理器、录音 + final int frameLength = 320; //录音视频帧长度为640 + final int sampleRate = 8000; //录音频采样率为8000 + List recordingAudioAllFrames = []; // 录制音频的所有帧 + List lockRecordingAudioAllFrames = []; // 录制音频的所有帧 + RxInt rotateAngle = 0.obs; // 旋转角度（以弧度为单位） + RxBool hasAudioData = false.obs; // 是否有音频数据 + RxInt lastAudioTimestamp = 0.obs; // 最后接收到的音频数据的时间戳 +} diff --git a/pubspec.lock b/pubspec.lock index f09c8feb..40584305 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -992,8 +992,8 @@ packages: dependency: "direct main" description: path: "." - ref: main - resolved-ref: aa93729f48762421658675800be68aee27b6d8fb + ref: "807ddb8e396c2dce16919df84efe795072404dde" + resolved-ref: "807ddb8e396c2dce16919df84efe795072404dde" url: "git@code-internal.star-lock.cn:StarlockTeam/jpush_flutter.git" source: git version: "2.5.8" diff --git a/pubspec.yaml b/pubspec.yaml index 9109e0a9..885185dc 100755 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -214,7 +214,7 @@ dependencies: jpush_flutter: git: url: git@code-internal.star-lock.cn:StarlockTeam/jpush_flutter.git - ref: main + ref: 807ddb8e396c2dce16919df84efe795072404dde #视频播放器 video_player: ^2.9.2 @@ -316,6 +316,7 @@ flutter: - images/lockType/ - assets/ - assets/html/h264.html + - assets/html/jmuxer.min.js - lan/ # An image asset can refer to one or more resolution-specific "variants", see # https://flutter.dev/assets-and-images/#resolution-aware