From cd39196257c8b34c2daa6d76bd3e4ea404df52f6 Mon Sep 17 00:00:00 2001 From: Jared Goodwin Date: Fri, 17 Jan 2020 07:47:54 -0800 Subject: [PATCH] Use chunking in WebRTC data channel. --- .../Communication/WebRtcSession.cs | 19 ++++++++++--- ScreenCast.Core/Models/FrameInfo.cs | 13 ++++++--- ScreenCast.Core/Models/Viewer.cs | 2 +- Server/wwwroot/scripts/Models/FrameInfo.ts | 1 + .../scripts/RemoteControl/RCBrowserSockets.js | 2 +- .../RemoteControl/RCBrowserSockets.js.map | 2 +- .../scripts/RemoteControl/RCBrowserSockets.ts | 2 +- .../scripts/RemoteControl/RtcSession.js | 23 ++++++++++------ .../scripts/RemoteControl/RtcSession.js.map | 2 +- .../scripts/RemoteControl/RtcSession.ts | 27 ++++++++++++------- 10 files changed, 63 insertions(+), 30 deletions(-) diff --git a/ScreenCast.Core/Communication/WebRtcSession.cs b/ScreenCast.Core/Communication/WebRtcSession.cs index cac3ea23..5f95a2fd 100644 --- a/ScreenCast.Core/Communication/WebRtcSession.cs +++ b/ScreenCast.Core/Communication/WebRtcSession.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Text; using System.Threading.Tasks; @@ -20,6 +21,7 @@ namespace Remotely.ScreenCast.Core.Communication public ulong CurrentBuffer { get; private set; } public bool IsDataChannelOpen => CaptureChannel?.State == DataChannel.ChannelState.Open; + public bool IsPeerConnected => PeerConnection?.IsConnected == true; private DataChannel CaptureChannel { get; set; } private PeerConnection PeerConnection { get; set; } public void AddIceCandidate(string sdpMid, int sdpMlineIndex, string candidate) @@ -35,8 +37,7 @@ namespace Remotely.ScreenCast.Core.Communication public async Task Init() { - CaptureChannel?.Dispose(); - PeerConnection?.Dispose(); + Logger.Debug("Starting WebRTC connection."); PeerConnection = new PeerConnection(); var config = new PeerConnectionConfiguration() @@ -62,13 +63,25 @@ namespace Remotely.ScreenCast.Core.Communication } public void SendCaptureFrame(int left, int top, int width, int height, byte[] imageBytes) { + for (var i = 0; i < imageBytes.Length; i += 50_000) + { + CaptureChannel.SendMessage(MessagePackSerializer.Serialize(new FrameInfo() + { + Left = left, + Top = top, + Width = width, + Height = height, + EndOfFrame = false, + ImageBytes = imageBytes.Skip(i).Take(50000).ToArray() + })); + } CaptureChannel.SendMessage(MessagePackSerializer.Serialize(new FrameInfo() { Left = left, Top = top, Width = width, Height = height, - ImageBytes = imageBytes + EndOfFrame = true })); } diff --git a/ScreenCast.Core/Models/FrameInfo.cs b/ScreenCast.Core/Models/FrameInfo.cs index d204a7bd..b438df7b 100644 --- a/ScreenCast.Core/Models/FrameInfo.cs +++ b/ScreenCast.Core/Models/FrameInfo.cs @@ -8,15 +8,20 @@ namespace Remotely.ScreenCast.Core.Models [MessagePackObject] public class FrameInfo { + [Key("EndOfFrame")] + public bool EndOfFrame { get; internal set; } + + [Key("Height")] + public int Height { get; set; } + + [Key("ImageBytes")] + public byte[] ImageBytes { get; set; } + [Key("Left")] public int Left { get; set; } [Key("Top")] public int Top { get; set; } [Key("Width")] public int Width { get; set; } - [Key("Height")] - public int Height { get; set; } - [Key("ImageBytes")] - public byte[] ImageBytes { get; set; } } } diff --git a/ScreenCast.Core/Models/Viewer.cs b/ScreenCast.Core/Models/Viewer.cs index 41a326ea..3f75692f 100644 --- a/ScreenCast.Core/Models/Viewer.cs +++ b/ScreenCast.Core/Models/Viewer.cs @@ -92,7 +92,7 @@ namespace Remotely.ScreenCast.Core.Models public bool IsUsingWebRtc() { - return RtcSession?.IsDataChannelOpen == true; + return RtcSession?.IsPeerConnected == true && RtcSession?.IsDataChannelOpen == true; } } } diff --git a/Server/wwwroot/scripts/Models/FrameInfo.ts b/Server/wwwroot/scripts/Models/FrameInfo.ts index 0b34153b..14bb4e1f 100644 --- a/Server/wwwroot/scripts/Models/FrameInfo.ts +++ b/Server/wwwroot/scripts/Models/FrameInfo.ts @@ -1,4 +1,5 @@ declare interface FrameInfo { + EndOfFrame: boolean; Left: number; Top: number; Width: number; diff --git a/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js b/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js index 08d79c40..eb8c07b8 100644 --- a/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js +++ b/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js @@ -139,7 +139,7 @@ export class RCBrowserSockets { UI.Screen2DContext.clearRect(0, 0, width, height); }); hubConnection.on("ScreenCapture", (buffer, left, top, width, height, captureTime) => { - console.log("Websocket frame received."); + //console.log("Websocket frame received."); this.SendLatencyUpdate(captureTime, buffer.byteLength); var url = window.URL.createObjectURL(new Blob([buffer])); var img = document.createElement("img"); diff --git a/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js.map b/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js.map index c1e9b2cc..9dea986a 100644 --- a/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js.map +++ b/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js.map @@ -1 +1 @@ -{"version":3,"file":"RCBrowserSockets.js","sourceRoot":"","sources":["RCBrowserSockets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC,IAAI,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;AAShC,MAAM,OAAO,gBAAgB;IAGzB,OAAO;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,OAAO,CAAC,oBAAoB,EAAE;aAC/C,OAAO,CAAC,eAAe,CAAC;aACxB,eAAe,CAAC,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;aACvE,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;aAC9C,KAAK,EAAE,CAAC;QAEb,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAE3C,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;YAC9B,IAAI,CAAC,6BAA6B,EAAE,CAAC;YACrC,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAC7C,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC;YACrC,EAAE,CAAC,YAAY,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YAC1C,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,EAAE,CAAC;QACpC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;YACX,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YAClC,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,qBAAqB,GAAG,CAAC,OAAO,EAAE,CAAC;YAChE,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAClF,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACjD,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;YACxC,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAClF,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACjD,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACP,CAAC;IAAA,CAAC;IAEF,gBAAgB,CAAC,SAA0B;QACvC,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,yBAAyB,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;SACrH;aACI;YACD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,yBAAyB,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;SAChE;IACL,CAAC;IACD,aAAa,CAAC,kBAAyC;QACnD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,sBAAsB,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAC3E,CAAC;IAGD,6BAA6B;QACzB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,+BAA+B,EAAE,aAAa,CAAC,QAAQ,EAAE,aAAa,CAAC,aAAa,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IACrI,CAAC;IACD,iBAAiB,CAAC,QAAc,EAAE,aAAqB;QACnD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,mBAAmB,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IACzE,CAAC;IACD,gBAAgB,CAAC,KAAa;QAC1B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IACD,aAAa,CAAC,QAAgB,EAAE,QAAgB;QAC5C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC5D,CAAC;IACD,aAAa,CAAC,MAAc,EAAE,QAAgB,EAAE,QAAgB;QAC5D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IACD,WAAW,CAAC,MAAc,EAAE,QAAgB,EAAE,QAAgB;QAC1D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IACD,aAAa;QACT,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC;IACD,aAAa;QACT,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC;IACD,aAAa,CAAC,KAAa,EAAE,KAAa;QACtC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IACD,WAAW;QACP,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,QAAgB,EAAE,QAAgB;QACtC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACtD,CAAC;IACD,cAAc,CAAC,MAAc,EAAE,MAAc;QACzC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IACD,WAAW,CAAC,GAAW;QACnB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAC3C,CAAC;IACD,SAAS,CAAC,GAAW;QACjB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IACD,YAAY,CAAC,GAAW;QACpB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IAC5C,CAAC;IAED,cAAc;QACV,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IACD,iBAAiB,CAAC,OAAe;QAC7B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,mBAAmB,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACrE,CAAC;IACD,iBAAiB,CAAC,YAAoB;QAClC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,mBAAmB,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC;IACD,qBAAqB,CAAC,IAAa;QAC/B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAC;IAC1D,CAAC;IACD,eAAe,CAAC,QAAiB;QAC7B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC;IACxD,CAAC;IAAA,CAAC;IACF,qBAAqB,CAAC,IAAY,EAAE,QAAiB;QACjD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,uBAAuB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IACO,oBAAoB,CAAC,aAAa;QACtC,aAAa,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,aAAqB,EAAE,EAAE;YAC/D,SAAS,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;YAC1C,EAAE,CAAC,yBAAyB,CAAC,KAAK,GAAG,aAAa,CAAC;YACnD,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,aAAa,EAAE,CAAC,kBAA0B,EAAE,WAAmB,EAAE,EAAE;YAChF,QAAQ,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC;YAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE;gBAClC,IAAI,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAC9C,MAAM,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,kBAAkB,EAAE;oBACzB,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;iBACnC;gBACD,QAAQ,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAC/D,MAAM,CAAC,OAAO,GAAG,CAAC,EAAc,EAAE,EAAE;oBAChC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;oBACzB,QAAQ,CAAC,gBAAgB,CAAC,yCAAyC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;wBAClF,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;oBACvC,CAAC,CAAC,CAAC;oBACF,EAAE,CAAC,aAAmC,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACrE,CAAC,CAAC;aACL;QACL,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,KAAa,EAAE,MAAc,EAAE,EAAE;YAC7D,EAAE,CAAC,YAAY,CAAC,KAAK,GAAG,KAAK,CAAC;YAC9B,EAAE,CAAC,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC;YAChC,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,MAAkB,EAAE,IAAW,EAAE,GAAU,EAAE,KAAY,EAAE,MAAa,EAAE,WAAiB,EAAE,EAAE;YAC9H,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;YACzC,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;YAEvD,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACzD,IAAI,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACxC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE;gBACd,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5D,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YACpC,CAAC,CAAC;YACF,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,aAAa,EAAE,CAAC,MAAkB,EAAE,EAAE;YACnD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YACtC,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAC7C,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,kCAAkC,CAAC;YAChE,EAAE,CAAC,WAAW,CAAC,uCAAuC,CAAC,CAAC;YACxD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,eAAe,EAAE,GAAG,EAAE;YACnC,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAC7C,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,0CAA0C,CAAC;YACxE,EAAE,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;YACjC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,mBAAmB,EAAE,GAAG,EAAE;YACvC,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAC7C,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,uBAAuB,CAAC;YACrD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAC9C,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,4BAA4B,CAAC;YAC1D,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,WAAmB,EAAE,EAAE;YAC3D,QAAQ,CAAC,KAAK,GAAG,GAAG,WAAW,qBAAqB,CAAC;QACzD,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,6BAA6B,EAAE,CAAC,WAAmB,EAAE,EAAE;YACpE,aAAa,CAAC,QAAQ,GAAG,WAAW,CAAC;YACrC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YAClC,EAAE,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,MAAkB,EAAE,EAAE;YACpD,IAAI,MAAM,CAAC,WAAW,EAAE;gBACpB,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC;aACrD;iBACI,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,EAAE;gBACxC,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC;aAC5C;iBACI;gBACD,IAAI,MAAM,GAAG,SAAS,CAAC,yBAAyB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBACpE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,8BAA8B,MAAM,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,CAAC;aAC5H;QACL,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC1C,EAAE,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,iBAAiB,EAAE,KAAK,EAAE,GAAW,EAAE,EAAE;YACtD,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;YACvC,aAAa,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YAChC,MAAM,aAAa,CAAC,UAAU,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAExD,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,SAAiB,EAAE,aAAqB,EAAE,MAAc,EAAE,EAAE;YACjG,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;YACvC,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC;gBACtC,SAAS,EAAE,SAAS;gBACpB,aAAa,EAAE,aAAa;gBAC5B,MAAM,EAAE,MAAM;aACV,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;IACP,CAAC;CACJ"} \ No newline at end of file +{"version":3,"file":"RCBrowserSockets.js","sourceRoot":"","sources":["RCBrowserSockets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC,IAAI,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;AAShC,MAAM,OAAO,gBAAgB;IAGzB,OAAO;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,OAAO,CAAC,oBAAoB,EAAE;aAC/C,OAAO,CAAC,eAAe,CAAC;aACxB,eAAe,CAAC,IAAI,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;aACvE,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;aAC9C,KAAK,EAAE,CAAC;QAEb,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAE3C,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;YAC9B,IAAI,CAAC,6BAA6B,EAAE,CAAC;YACrC,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAC7C,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC;YACrC,EAAE,CAAC,YAAY,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;YAC1C,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,EAAE,CAAC;QACpC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;YACX,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YAClC,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,qBAAqB,GAAG,CAAC,OAAO,EAAE,CAAC;YAChE,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAClF,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACjD,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;YACxC,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAClF,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACjD,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACP,CAAC;IAAA,CAAC;IAEF,gBAAgB,CAAC,SAA0B;QACvC,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,yBAAyB,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;SACrH;aACI;YACD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,yBAAyB,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;SAChE;IACL,CAAC;IACD,aAAa,CAAC,kBAAyC;QACnD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,sBAAsB,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAC3E,CAAC;IAGD,6BAA6B;QACzB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,+BAA+B,EAAE,aAAa,CAAC,QAAQ,EAAE,aAAa,CAAC,aAAa,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IACrI,CAAC;IACD,iBAAiB,CAAC,QAAc,EAAE,aAAqB;QACnD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,mBAAmB,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IACzE,CAAC;IACD,gBAAgB,CAAC,KAAa;QAC1B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IACD,aAAa,CAAC,QAAgB,EAAE,QAAgB;QAC5C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC5D,CAAC;IACD,aAAa,CAAC,MAAc,EAAE,QAAgB,EAAE,QAAgB;QAC5D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IACD,WAAW,CAAC,MAAc,EAAE,QAAgB,EAAE,QAAgB;QAC1D,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;IACD,aAAa;QACT,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC;IACD,aAAa;QACT,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC;IACD,aAAa,CAAC,KAAa,EAAE,KAAa;QACtC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IACD,WAAW;QACP,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,QAAgB,EAAE,QAAgB;QACtC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACtD,CAAC;IACD,cAAc,CAAC,MAAc,EAAE,MAAc;QACzC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IACD,WAAW,CAAC,GAAW;QACnB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAC3C,CAAC;IACD,SAAS,CAAC,GAAW;QACjB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IACD,YAAY,CAAC,GAAW;QACpB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IAC5C,CAAC;IAED,cAAc;QACV,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IACD,iBAAiB,CAAC,OAAe;QAC7B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,mBAAmB,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACrE,CAAC;IACD,iBAAiB,CAAC,YAAoB;QAClC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,mBAAmB,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC;IACD,qBAAqB,CAAC,IAAa;QAC/B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAC;IAC1D,CAAC;IACD,eAAe,CAAC,QAAiB;QAC7B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC;IACxD,CAAC;IAAA,CAAC;IACF,qBAAqB,CAAC,IAAY,EAAE,QAAiB;QACjD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,uBAAuB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IACO,oBAAoB,CAAC,aAAa;QACtC,aAAa,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,aAAqB,EAAE,EAAE;YAC/D,SAAS,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;YAC1C,EAAE,CAAC,yBAAyB,CAAC,KAAK,GAAG,aAAa,CAAC;YACnD,YAAY,CAAC,oBAAoB,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,aAAa,EAAE,CAAC,kBAA0B,EAAE,WAAmB,EAAE,EAAE;YAChF,QAAQ,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC;YAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE;gBAClC,IAAI,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAC9C,MAAM,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;gBAC9C,IAAI,CAAC,IAAI,kBAAkB,EAAE;oBACzB,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;iBACnC;gBACD,QAAQ,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAC/D,MAAM,CAAC,OAAO,GAAG,CAAC,EAAc,EAAE,EAAE;oBAChC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;oBACzB,QAAQ,CAAC,gBAAgB,CAAC,yCAAyC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;wBAClF,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;oBACvC,CAAC,CAAC,CAAC;oBACF,EAAE,CAAC,aAAmC,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACrE,CAAC,CAAC;aACL;QACL,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,KAAa,EAAE,MAAc,EAAE,EAAE;YAC7D,EAAE,CAAC,YAAY,CAAC,KAAK,GAAG,KAAK,CAAC;YAC9B,EAAE,CAAC,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC;YAChC,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,MAAkB,EAAE,IAAW,EAAE,GAAU,EAAE,KAAY,EAAE,MAAa,EAAE,WAAiB,EAAE,EAAE;YAC9H,2CAA2C;YAC3C,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;YAEvD,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACzD,IAAI,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACxC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE;gBACd,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5D,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YACpC,CAAC,CAAC;YACF,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,aAAa,EAAE,CAAC,MAAkB,EAAE,EAAE;YACnD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YACtC,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAC7C,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,kCAAkC,CAAC;YAChE,EAAE,CAAC,WAAW,CAAC,uCAAuC,CAAC,CAAC;YACxD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,eAAe,EAAE,GAAG,EAAE;YACnC,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAC7C,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,0CAA0C,CAAC;YACxE,EAAE,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;YACjC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,mBAAmB,EAAE,GAAG,EAAE;YACvC,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAC7C,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,uBAAuB,CAAC;YACrD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAC9C,EAAE,CAAC,aAAa,CAAC,SAAS,GAAG,4BAA4B,CAAC;YAC1D,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,WAAmB,EAAE,EAAE;YAC3D,QAAQ,CAAC,KAAK,GAAG,GAAG,WAAW,qBAAqB,CAAC;QACzD,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,6BAA6B,EAAE,CAAC,WAAmB,EAAE,EAAE;YACpE,aAAa,CAAC,QAAQ,GAAG,WAAW,CAAC;YACrC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YAClC,EAAE,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,MAAkB,EAAE,EAAE;YACpD,IAAI,MAAM,CAAC,WAAW,EAAE;gBACpB,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC;aACrD;iBACI,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,EAAE;gBACxC,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC;aAC5C;iBACI;gBACD,IAAI,MAAM,GAAG,SAAS,CAAC,yBAAyB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBACpE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,8BAA8B,MAAM,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,CAAC;aAC5H;QACL,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC1C,EAAE,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,aAAa,CAAC,EAAE,CAAC,iBAAiB,EAAE,KAAK,EAAE,GAAW,EAAE,EAAE;YACtD,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;YACvC,aAAa,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YAChC,MAAM,aAAa,CAAC,UAAU,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAExD,CAAC,CAAC,CAAC;QACH,aAAa,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,SAAiB,EAAE,aAAqB,EAAE,MAAc,EAAE,EAAE;YACjG,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;YACvC,aAAa,CAAC,UAAU,CAAC,gBAAgB,CAAC;gBACtC,SAAS,EAAE,SAAS;gBACpB,aAAa,EAAE,aAAa;gBAC5B,MAAM,EAAE,MAAM;aACV,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;IACP,CAAC;CACJ"} \ No newline at end of file diff --git a/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.ts b/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.ts index aa9863da..4e9c589e 100644 --- a/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.ts +++ b/Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.ts @@ -155,7 +155,7 @@ export class RCBrowserSockets { UI.Screen2DContext.clearRect(0, 0, width, height); }); hubConnection.on("ScreenCapture", (buffer: Uint8Array, left:number, top:number, width:number, height:number, captureTime: Date) => { - console.log("Websocket frame received."); + //console.log("Websocket frame received."); this.SendLatencyUpdate(captureTime, buffer.byteLength); var url = window.URL.createObjectURL(new Blob([buffer])); diff --git a/Server/wwwroot/scripts/RemoteControl/RtcSession.js b/Server/wwwroot/scripts/RemoteControl/RtcSession.js index b00a6223..ec7d34b9 100644 --- a/Server/wwwroot/scripts/RemoteControl/RtcSession.js +++ b/Server/wwwroot/scripts/RemoteControl/RtcSession.js @@ -4,6 +4,7 @@ import { RemoteControl } from "./Main.js"; export class RtcSession { constructor() { this.MessagePack = window['MessagePack']; + this.PartialFrames = []; } Init() { this.PeerConnection = new RTCPeerConnection({ @@ -29,15 +30,21 @@ export class RtcSession { if (ev.data.arrayBuffer) { data = await ev.data.arrayBuffer(); } - console.log("WebRTC frame received. Size: " + data.byteLength); + //console.log("WebRTC frame received. Size: " + data.byteLength); var frameInfo = this.MessagePack.decode(data); - var url = window.URL.createObjectURL(new Blob([frameInfo.ImageBytes])); - var img = document.createElement("img"); - img.onload = () => { - UI.Screen2DContext.drawImage(img, frameInfo.Left, frameInfo.Top, frameInfo.Width, frameInfo.Height); - window.URL.revokeObjectURL(url); - }; - img.src = url; + if (frameInfo.EndOfFrame) { + var url = window.URL.createObjectURL(new Blob(this.PartialFrames)); + var img = document.createElement("img"); + img.onload = () => { + UI.Screen2DContext.drawImage(img, frameInfo.Left, frameInfo.Top, frameInfo.Width, frameInfo.Height); + window.URL.revokeObjectURL(url); + }; + img.src = url; + this.PartialFrames = []; + } + else { + this.PartialFrames.push(frameInfo.ImageBytes); + } }; this.DataChannel.onopen = (ev) => { console.log("Data channel opened."); diff --git a/Server/wwwroot/scripts/RemoteControl/RtcSession.js.map b/Server/wwwroot/scripts/RemoteControl/RtcSession.js.map index 9f5ed326..92c1abba 100644 --- a/Server/wwwroot/scripts/RemoteControl/RtcSession.js.map +++ b/Server/wwwroot/scripts/RemoteControl/RtcSession.js.map @@ -1 +1 @@ -{"version":3,"file":"RtcSession.js","sourceRoot":"","sources":["RtcSession.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAG1C,MAAM,OAAO,UAAU;IAAvB;QAGI,gBAAW,GAAQ,MAAM,CAAC,aAAa,CAAC,CAAC;IAyE7C,CAAC;IAxEG,IAAI;QACA,IAAI,CAAC,cAAc,GAAG,IAAI,iBAAiB,CAAC;YACxC,UAAU,EAAE;gBACR,EAAE,IAAI,EAAE,+BAA+B,EAAE;gBACzC,EAAE,IAAI,EAAE,+BAA+B,EAAE;aAC5C;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,cAAc,CAAC,aAAa,GAAG,CAAC,EAAE,EAAE,EAAE;YACvC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;YACtC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC;YAC9B,IAAI,CAAC,WAAW,CAAC,mBAAmB,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC1C,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YACtC,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC9B,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACxC,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC9B,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;YACjD,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,SAAS,GAAG,KAAK,EAAE,EAAE,EAAE,EAAE;gBACtC,IAAI,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC;gBAEnB,IAAI,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE;oBACrB,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;iBACtC;gBACD,OAAO,CAAC,GAAG,CAAC,+BAA+B,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC/D,IAAI,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAc,CAAC;gBAC3D,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBACvE,IAAI,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;gBACxC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE;oBACd,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;oBACpG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBACpC,CAAC,CAAC;gBACF,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;YAClB,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACxC,CAAC,CAAC;QACN,CAAC,CAAC;QACF,IAAI,CAAC,cAAc,CAAC,uBAAuB,GAAG,UAAU,EAAE;YACtD,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;QACvE,CAAC,CAAA;QAED,IAAI,CAAC,cAAc,CAAC,0BAA0B,GAAG,UAAU,EAAE;YACzD,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1E,CAAC,CAAA;QACD,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,KAAK,EAAE,EAAE,EAAE,EAAE;YAC9C,MAAM,aAAa,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QACxE,CAAC,CAAC;IACN,CAAC;IACD,UAAU;QACN,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;IACD,KAAK,CAAC,eAAe,CAAC,GAAW;QAC7B,MAAM,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAE5E,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,OAAO,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,EAAE,CAAC,CAAC;YACxF,MAAM,aAAa,CAAC,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAC7F,CAAC,CAAC,CAAA;IACN,CAAC;IACD,KAAK,CAAC,gBAAgB,CAAC,SAA0B;QAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,OAAO,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAA;IACN,CAAC;CACJ"} \ No newline at end of file +{"version":3,"file":"RtcSession.js","sourceRoot":"","sources":["RtcSession.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,OAAO,UAAU;IAAvB;QAGI,gBAAW,GAAQ,MAAM,CAAC,aAAa,CAAC,CAAC;QACzC,kBAAa,GAAiB,EAAE,CAAC;IAgFrC,CAAC;IA/EG,IAAI;QACA,IAAI,CAAC,cAAc,GAAG,IAAI,iBAAiB,CAAC;YACxC,UAAU,EAAE;gBACR,EAAE,IAAI,EAAE,+BAA+B,EAAE;gBACzC,EAAE,IAAI,EAAE,+BAA+B,EAAE;aAC5C;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,cAAc,CAAC,aAAa,GAAG,CAAC,EAAE,EAAE,EAAE;YACvC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;YACtC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC;YAC9B,IAAI,CAAC,WAAW,CAAC,mBAAmB,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC1C,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YACtC,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC9B,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACxC,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC9B,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;YACjD,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,SAAS,GAAG,KAAK,EAAE,EAAE,EAAE,EAAE;gBACtC,IAAI,IAAI,GAAG,EAAE,CAAC,IAAmB,CAAC;gBAElC,IAAI,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE;oBACrB,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;iBACtC;gBACD,iEAAiE;gBACjE,IAAI,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAc,CAAC;gBAC3D,IAAI,SAAS,CAAC,UAAU,EAAE;oBACtB,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;oBACnE,IAAI,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBACxC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE;wBACd,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;wBACpG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBACpC,CAAC,CAAC;oBACF,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;oBACd,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;iBAC3B;qBACI;oBACD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;iBACjD;YAEL,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACxC,CAAC,CAAC;QACN,CAAC,CAAC;QACF,IAAI,CAAC,cAAc,CAAC,uBAAuB,GAAG,UAAU,EAAE;YACtD,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;QACvE,CAAC,CAAA;QAED,IAAI,CAAC,cAAc,CAAC,0BAA0B,GAAG,UAAU,EAAE;YACzD,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1E,CAAC,CAAA;QACD,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,KAAK,EAAE,EAAE,EAAE,EAAE;YAC9C,MAAM,aAAa,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QACxE,CAAC,CAAC;IACN,CAAC;IACD,UAAU;QACN,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;IACD,KAAK,CAAC,eAAe,CAAC,GAAW;QAC7B,MAAM,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAE5E,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,OAAO,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,EAAE,CAAC,CAAC;YACxF,MAAM,aAAa,CAAC,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAC7F,CAAC,CAAC,CAAA;IACN,CAAC;IACD,KAAK,CAAC,gBAAgB,CAAC,SAA0B;QAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,OAAO,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAA;IACN,CAAC;CACJ"} \ No newline at end of file diff --git a/Server/wwwroot/scripts/RemoteControl/RtcSession.ts b/Server/wwwroot/scripts/RemoteControl/RtcSession.ts index 8c67566a..a3c0efde 100644 --- a/Server/wwwroot/scripts/RemoteControl/RtcSession.ts +++ b/Server/wwwroot/scripts/RemoteControl/RtcSession.ts @@ -2,11 +2,11 @@ import * as Utilities from "../Utilities.js"; import { RemoteControl } from "./Main.js"; - export class RtcSession { PeerConnection: RTCPeerConnection; DataChannel: RTCDataChannel; MessagePack: any = window['MessagePack']; + PartialFrames: Uint8Array[] = []; Init() { this.PeerConnection = new RTCPeerConnection({ iceServers: [ @@ -28,20 +28,27 @@ export class RtcSession { console.log("Data channel error.", ev.error); }; this.DataChannel.onmessage = async (ev) => { - var data = ev.data; + var data = ev.data as ArrayBuffer; if (ev.data.arrayBuffer) { data = await ev.data.arrayBuffer(); } - console.log("WebRTC frame received. Size: " + data.byteLength); + //console.log("WebRTC frame received. Size: " + data.byteLength); var frameInfo = this.MessagePack.decode(data) as FrameInfo; - var url = window.URL.createObjectURL(new Blob([frameInfo.ImageBytes])); - var img = document.createElement("img"); - img.onload = () => { - UI.Screen2DContext.drawImage(img, frameInfo.Left, frameInfo.Top, frameInfo.Width, frameInfo.Height); - window.URL.revokeObjectURL(url); - }; - img.src = url; + if (frameInfo.EndOfFrame) { + var url = window.URL.createObjectURL(new Blob(this.PartialFrames)); + var img = document.createElement("img"); + img.onload = () => { + UI.Screen2DContext.drawImage(img, frameInfo.Left, frameInfo.Top, frameInfo.Width, frameInfo.Height); + window.URL.revokeObjectURL(url); + }; + img.src = url; + this.PartialFrames = []; + } + else { + this.PartialFrames.push(frameInfo.ImageBytes); + } + }; this.DataChannel.onopen = (ev) => { console.log("Data channel opened.");