mirrors/Remotely
3
0
Fork 0
mirror of https://github.com/immense/Remotely.git synced 2025-10-26 11:27:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use chunking in WebRTC data channel.

Browse Source
This commit is contained in:
Jared Goodwin 2020-01-17 07:47:54 -08:00
parent 3082ed9773
commit cd39196257
10 changed files with 63 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
ScreenCast.Core/Communication/WebRtcSession.cs
View File

@ -6,6 +6,7 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
@ -20,6 +21,7 @@ namespace Remotely.ScreenCast.Core.Communication
public ulong CurrentBuffer { get; private set; }
public bool IsDataChannelOpen => CaptureChannel?.State == DataChannel.ChannelState.Open;
public bool IsPeerConnected => PeerConnection?.IsConnected == true;
private DataChannel CaptureChannel { get; set; }
private PeerConnection PeerConnection { get; set; }
public void AddIceCandidate(string sdpMid, int sdpMlineIndex, string candidate)
@ -35,8 +37,7 @@ namespace Remotely.ScreenCast.Core.Communication
public async Task Init()
{
CaptureChannel?.Dispose();
PeerConnection?.Dispose();
Logger.Debug("Starting WebRTC connection.");
PeerConnection = new PeerConnection();
var config = new PeerConnectionConfiguration()
@ -62,13 +63,25 @@ namespace Remotely.ScreenCast.Core.Communication
}
public void SendCaptureFrame(int left, int top, int width, int height, byte[] imageBytes)
{
for (var i = 0; i < imageBytes.Length; i += 50_000)
{
CaptureChannel.SendMessage(MessagePackSerializer.Serialize(new FrameInfo()
{
Left = left,
Top = top,
Width = width,
Height = height,
EndOfFrame = false,
ImageBytes = imageBytes.Skip(i).Take(50000).ToArray()
}));
}
CaptureChannel.SendMessage(MessagePackSerializer.Serialize(new FrameInfo()
{
Left = left,
Top = top,
Width = width,
Height = height,
ImageBytes = imageBytes
EndOfFrame = true
}));
}

13
ScreenCast.Core/Models/FrameInfo.cs
View File

@ -8,15 +8,20 @@ namespace Remotely.ScreenCast.Core.Models
[MessagePackObject]
public class FrameInfo
{
[Key("EndOfFrame")]
public bool EndOfFrame { get; internal set; }
[Key("Height")]
public int Height { get; set; }
[Key("ImageBytes")]
public byte[] ImageBytes { get; set; }
[Key("Left")]
public int Left { get; set; }
[Key("Top")]
public int Top { get; set; }
[Key("Width")]
public int Width { get; set; }
[Key("Height")]
public int Height { get; set; }
[Key("ImageBytes")]
public byte[] ImageBytes { get; set; }
}
}

2
ScreenCast.Core/Models/Viewer.cs
View File

@ -92,7 +92,7 @@ namespace Remotely.ScreenCast.Core.Models
public bool IsUsingWebRtc()
{
return RtcSession?.IsDataChannelOpen == true;
return RtcSession?.IsPeerConnected == true && RtcSession?.IsDataChannelOpen == true;
}
}
}

1
Server/wwwroot/scripts/Models/FrameInfo.ts
View File

@ -1,4 +1,5 @@
declare interface FrameInfo {
EndOfFrame: boolean;
Left: number;
Top: number;
Width: number;

2
Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js
View File

@ -139,7 +139,7 @@ export class RCBrowserSockets {
UI.Screen2DContext.clearRect(0, 0, width, height);
});
hubConnection.on("ScreenCapture", (buffer, left, top, width, height, captureTime) => {
console.log("Websocket frame received.");
//console.log("Websocket frame received.");
this.SendLatencyUpdate(captureTime, buffer.byteLength);
var url = window.URL.createObjectURL(new Blob([buffer]));
var img = document.createElement("img");

2
Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js.map
View File

File diff suppressed because one or more lines are too long

2
Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.ts
View File

@ -155,7 +155,7 @@ export class RCBrowserSockets {
UI.Screen2DContext.clearRect(0, 0, width, height);
});
hubConnection.on("ScreenCapture", (buffer: Uint8Array, left:number, top:number, width:number, height:number, captureTime: Date) => {
console.log("Websocket frame received.");
//console.log("Websocket frame received.");
this.SendLatencyUpdate(captureTime, buffer.byteLength);
var url = window.URL.createObjectURL(new Blob([buffer]));

23
Server/wwwroot/scripts/RemoteControl/RtcSession.js
View File

@ -4,6 +4,7 @@ import { RemoteControl } from "./Main.js";
export class RtcSession {
constructor() {
this.MessagePack = window['MessagePack'];
this.PartialFrames = [];
}
Init() {
this.PeerConnection = new RTCPeerConnection({
@ -29,15 +30,21 @@ export class RtcSession {
if (ev.data.arrayBuffer) {
data = await ev.data.arrayBuffer();
}
console.log("WebRTC frame received. Size: " + data.byteLength);
//console.log("WebRTC frame received. Size: " + data.byteLength);
var frameInfo = this.MessagePack.decode(data);
var url = window.URL.createObjectURL(new Blob([frameInfo.ImageBytes]));
var img = document.createElement("img");
img.onload = () => {
UI.Screen2DContext.drawImage(img, frameInfo.Left, frameInfo.Top, frameInfo.Width, frameInfo.Height);
window.URL.revokeObjectURL(url);
};
img.src = url;
if (frameInfo.EndOfFrame) {
var url = window.URL.createObjectURL(new Blob(this.PartialFrames));
var img = document.createElement("img");
img.onload = () => {
UI.Screen2DContext.drawImage(img, frameInfo.Left, frameInfo.Top, frameInfo.Width, frameInfo.Height);
window.URL.revokeObjectURL(url);
};
img.src = url;
this.PartialFrames = [];
}
else {
this.PartialFrames.push(frameInfo.ImageBytes);
}
};
this.DataChannel.onopen = (ev) => {
console.log("Data channel opened.");

2
Server/wwwroot/scripts/RemoteControl/RtcSession.js.map
View File

@ -1 +1 @@
{"version":3,"file":"RtcSession.js","sourceRoot":"","sources":["RtcSession.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAG1C,MAAM,OAAO,UAAU;IAAvB;QAGI,gBAAW,GAAQ,MAAM,CAAC,aAAa,CAAC,CAAC;IAyE7C,CAAC;IAxEG,IAAI;QACA,IAAI,CAAC,cAAc,GAAG,IAAI,iBAAiB,CAAC;YACxC,UAAU,EAAE;gBACR,EAAE,IAAI,EAAE,+BAA+B,EAAE;gBACzC,EAAE,IAAI,EAAE,+BAA+B,EAAE;aAC5C;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,cAAc,CAAC,aAAa,GAAG,CAAC,EAAE,EAAE,EAAE;YACvC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;YACtC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC;YAC9B,IAAI,CAAC,WAAW,CAAC,mBAAmB,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC1C,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YACtC,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC9B,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACxC,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC9B,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;YACjD,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,SAAS,GAAG,KAAK,EAAE,EAAE,EAAE,EAAE;gBACtC,IAAI,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC;gBAEnB,IAAI,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE;oBACrB,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;iBACtC;gBACD,OAAO,CAAC,GAAG,CAAC,+BAA+B,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC/D,IAAI,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAc,CAAC;gBAC3D,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBACvE,IAAI,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;gBACxC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE;oBACd,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;oBACpG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;gBACpC,CAAC,CAAC;gBACF,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;YAClB,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACxC,CAAC,CAAC;QACN,CAAC,CAAC;QACF,IAAI,CAAC,cAAc,CAAC,uBAAuB,GAAG,UAAU,EAAE;YACtD,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;QACvE,CAAC,CAAA;QAED,IAAI,CAAC,cAAc,CAAC,0BAA0B,GAAG,UAAU,EAAE;YACzD,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1E,CAAC,CAAA;QACD,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,KAAK,EAAE,EAAE,EAAE,EAAE;YAC9C,MAAM,aAAa,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QACxE,CAAC,CAAC;IACN,CAAC;IACD,UAAU;QACN,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;IACD,KAAK,CAAC,eAAe,CAAC,GAAW;QAC7B,MAAM,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAE5E,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,OAAO,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,EAAE,CAAC,CAAC;YACxF,MAAM,aAAa,CAAC,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAC7F,CAAC,CAAC,CAAA;IACN,CAAC;IACD,KAAK,CAAC,gBAAgB,CAAC,SAA0B;QAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,OAAO,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAA;IACN,CAAC;CACJ"}
{"version":3,"file":"RtcSession.js","sourceRoot":"","sources":["RtcSession.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,OAAO,UAAU;IAAvB;QAGI,gBAAW,GAAQ,MAAM,CAAC,aAAa,CAAC,CAAC;QACzC,kBAAa,GAAiB,EAAE,CAAC;IAgFrC,CAAC;IA/EG,IAAI;QACA,IAAI,CAAC,cAAc,GAAG,IAAI,iBAAiB,CAAC;YACxC,UAAU,EAAE;gBACR,EAAE,IAAI,EAAE,+BAA+B,EAAE;gBACzC,EAAE,IAAI,EAAE,+BAA+B,EAAE;aAC5C;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,cAAc,CAAC,aAAa,GAAG,CAAC,EAAE,EAAE,EAAE;YACvC,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;YACtC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC;YAC9B,IAAI,CAAC,WAAW,CAAC,mBAAmB,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC1C,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YACtC,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC9B,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACxC,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,OAAO,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC9B,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;YACjD,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,SAAS,GAAG,KAAK,EAAE,EAAE,EAAE,EAAE;gBACtC,IAAI,IAAI,GAAG,EAAE,CAAC,IAAmB,CAAC;gBAElC,IAAI,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE;oBACrB,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;iBACtC;gBACD,iEAAiE;gBACjE,IAAI,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAc,CAAC;gBAC3D,IAAI,SAAS,CAAC,UAAU,EAAE;oBACtB,IAAI,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;oBACnE,IAAI,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;oBACxC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE;wBACd,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;wBACpG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;oBACpC,CAAC,CAAC;oBACF,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;oBACd,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;iBAC3B;qBACI;oBACD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;iBACjD;YAEL,CAAC,CAAC;YACF,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,EAAE;gBAC7B,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACxC,CAAC,CAAC;QACN,CAAC,CAAC;QACF,IAAI,CAAC,cAAc,CAAC,uBAAuB,GAAG,UAAU,EAAE;YACtD,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;QACvE,CAAC,CAAA;QAED,IAAI,CAAC,cAAc,CAAC,0BAA0B,GAAG,UAAU,EAAE;YACzD,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1E,CAAC,CAAA;QACD,IAAI,CAAC,cAAc,CAAC,cAAc,GAAG,KAAK,EAAE,EAAE,EAAE,EAAE;YAC9C,MAAM,aAAa,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QACxE,CAAC,CAAC;IACN,CAAC;IACD,UAAU;QACN,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;IACD,KAAK,CAAC,eAAe,CAAC,GAAW;QAC7B,MAAM,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QAE5E,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,OAAO,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,EAAE,CAAC,CAAC;YACxF,MAAM,aAAa,CAAC,gBAAgB,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAC7F,CAAC,CAAC,CAAA;IACN,CAAC;IACD,KAAK,CAAC,gBAAgB,CAAC,SAA0B;QAC7C,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,OAAO,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAA;IACN,CAAC;CACJ"}

27
Server/wwwroot/scripts/RemoteControl/RtcSession.ts
View File

@ -2,11 +2,11 @@
import * as Utilities from "../Utilities.js";
import { RemoteControl } from "./Main.js";
export class RtcSession {
PeerConnection: RTCPeerConnection;
DataChannel: RTCDataChannel;
MessagePack: any = window['MessagePack'];
PartialFrames: Uint8Array[] = [];
Init() {
this.PeerConnection = new RTCPeerConnection({
iceServers: [
@ -28,20 +28,27 @@ export class RtcSession {
console.log("Data channel error.", ev.error);
};
this.DataChannel.onmessage = async (ev) => {
var data = ev.data;
var data = ev.data as ArrayBuffer;
if (ev.data.arrayBuffer) {
data = await ev.data.arrayBuffer();
}
console.log("WebRTC frame received. Size: " + data.byteLength);
//console.log("WebRTC frame received. Size: " + data.byteLength);
var frameInfo = this.MessagePack.decode(data) as FrameInfo;
var url = window.URL.createObjectURL(new Blob([frameInfo.ImageBytes]));
var img = document.createElement("img");
img.onload = () => {
UI.Screen2DContext.drawImage(img, frameInfo.Left, frameInfo.Top, frameInfo.Width, frameInfo.Height);
window.URL.revokeObjectURL(url);
};
img.src = url;
if (frameInfo.EndOfFrame) {
var url = window.URL.createObjectURL(new Blob(this.PartialFrames));
var img = document.createElement("img");
img.onload = () => {
UI.Screen2DContext.drawImage(img, frameInfo.Left, frameInfo.Top, frameInfo.Width, frameInfo.Height);
window.URL.revokeObjectURL(url);
};
img.src = url;
this.PartialFrames = [];
}
else {
this.PartialFrames.push(frameInfo.ImageBytes);
}
};
this.DataChannel.onopen = (ev) => {
console.log("Data channel opened.");