From dd3b9083bdf502fc78fb95dbbb1d90351a1c2d99 Mon Sep 17 00:00:00 2001 From: Jared Goodwin Date: Thu, 6 Aug 2020 17:20:53 -0700 Subject: [PATCH] Update WebRTC library and accomodate API changes. --- Desktop.Core/Desktop.Core.csproj | 2 +- Desktop.Core/Models/Viewer.cs | 6 +- Desktop.Core/Services/WebRtcSession.cs | 121 ++++++++++++++---- Desktop.Core/Services/WebRtcSessionFactory.cs | 2 +- Desktop.Linux/Desktop.Linux.csproj | 6 +- Desktop.Win/Desktop.Win.csproj | 2 +- Desktop.Win/Services/ScreenCapturerWin.cs | 11 +- Server/Server.csproj | 2 +- Tests/Tests.csproj | 2 +- 9 files changed, 107 insertions(+), 47 deletions(-) diff --git a/Desktop.Core/Desktop.Core.csproj b/Desktop.Core/Desktop.Core.csproj index 06c7c67c..1679f2ba 100644 --- a/Desktop.Core/Desktop.Core.csproj +++ b/Desktop.Core/Desktop.Core.csproj @@ -38,7 +38,7 @@ - + diff --git a/Desktop.Core/Models/Viewer.cs b/Desktop.Core/Models/Viewer.cs index f28b98a2..0d53aa2e 100644 --- a/Desktop.Core/Models/Viewer.cs +++ b/Desktop.Core/Models/Viewer.cs @@ -92,11 +92,11 @@ namespace Remotely.Desktop.Core.Models RtcSession = WebRtcSessionFactory.GetNewSession(this); RtcSession.LocalSdpReady += async (sender, sdp) => { - await CasterSocket.SendRtcOfferToBrowser(sdp, ViewerConnectionID, iceServers); + await CasterSocket.SendRtcOfferToBrowser(sdp.Content, ViewerConnectionID, iceServers); }; - RtcSession.IceCandidateReady += async (sender, args) => + RtcSession.IceCandidateReady += async (sender, candidate) => { - await CasterSocket.SendIceCandidateToBrowser(args.candidate, args.sdpMlineIndex, args.sdpMid, ViewerConnectionID); + await CasterSocket.SendIceCandidateToBrowser(candidate.Content, candidate.SdpMlineIndex, candidate.SdpMid, ViewerConnectionID); }; await RtcSession.Init(iceServers); diff --git a/Desktop.Core/Services/WebRtcSession.cs b/Desktop.Core/Services/WebRtcSession.cs index 2aa3f13a..65024ce8 100644 --- a/Desktop.Core/Services/WebRtcSession.cs +++ b/Desktop.Core/Services/WebRtcSession.cs @@ -1,10 +1,12 @@ using MessagePack; using Microsoft.MixedReality.WebRTC; +using Remotely.Desktop.Core.Models; using Remotely.Shared.Models; using Remotely.Shared.Models.RtcDtos; using Remotely.Shared.Utilities; using System; using System.Collections.Generic; +using System.Drawing; using System.Linq; using System.Threading.Tasks; @@ -12,31 +14,40 @@ namespace Remotely.Desktop.Core.Services { public class WebRtcSession : IDisposable { - public WebRtcSession(IRtcMessageHandler rtcMessageHandler) + public WebRtcSession(Viewer viewer, IRtcMessageHandler rtcMessageHandler) { + Viewer = viewer; RtcMessageHandler = rtcMessageHandler; } - public event EventHandler<(string candidate, int sdpMlineIndex, string sdpMid)> IceCandidateReady; + public event EventHandler IceCandidateReady; - public event EventHandler LocalSdpReady; + public event EventHandler LocalSdpReady; public ulong CurrentBuffer { get; private set; } public bool IsDataChannelOpen => CaptureChannel?.State == DataChannel.ChannelState.Open; - public bool IsPeerConnected => PeerConnection?.IsConnected == true; + public bool IsPeerConnected => PeerSession?.IsConnected == true; private DataChannel CaptureChannel { get; set; } private IceServerModel[] IceServers { get; set; } - private PeerConnection PeerConnection { get; set; } + private PeerConnection PeerSession { get; set; } private IRtcMessageHandler RtcMessageHandler { get; } + private Transceiver Transceiver { get; set; } + private ExternalVideoTrackSource VideoSource { get; set; } + private Viewer Viewer { get; } + public void AddIceCandidate(string sdpMid, int sdpMlineIndex, string candidate) { - PeerConnection.AddIceCandidate(sdpMid, sdpMlineIndex, candidate); + PeerSession.AddIceCandidate(new IceCandidate() + { + Content = candidate, + SdpMid = sdpMid, + SdpMlineIndex = sdpMlineIndex + }); } public void Dispose() { - CaptureChannel?.Dispose(); - PeerConnection?.Dispose(); + PeerSession?.Dispose(); } public async Task Init(IceServerModel[] iceServers) @@ -45,7 +56,7 @@ namespace Remotely.Desktop.Core.Services IceServers = iceServers; - PeerConnection = new PeerConnection(); + PeerSession = new PeerConnection(); var iceList = IceServers.Select(x => new IceServer() { @@ -59,18 +70,28 @@ namespace Remotely.Desktop.Core.Services IceServers = iceList }; - await PeerConnection.InitializeAsync(config); + await PeerSession.InitializeAsync(config); - PeerConnection.LocalSdpReadytoSend += PeerConnection_LocalSdpReadytoSend; - PeerConnection.Connected += PeerConnection_Connected; - PeerConnection.IceStateChanged += PeerConnection_IceStateChanged; - PeerConnection.IceCandidateReadytoSend += PeerConnection_IceCandidateReadytoSend; - CaptureChannel = await PeerConnection.AddDataChannelAsync("ScreenCapture", true, true); + PeerSession.LocalSdpReadytoSend += PeerSession_LocalSdpReadytoSend; ; + PeerSession.Connected += PeerConnection_Connected; + PeerSession.IceStateChanged += PeerConnection_IceStateChanged; + PeerSession.IceCandidateReadytoSend += PeerSession_IceCandidateReadytoSend; ; + + CaptureChannel = await PeerSession.AddDataChannelAsync("ScreenCapture", true, true); CaptureChannel.BufferingChanged += DataChannel_BufferingChanged; CaptureChannel.MessageReceived += CaptureChannel_MessageReceived; CaptureChannel.StateChanged += CaptureChannel_StateChanged; - PeerConnection.CreateOffer(); + + //VideoSource = ExternalVideoTrackSource.CreateFromArgb32Callback(GetCaptureFrame); + //Transceiver = PeerSession.AddTransceiver(MediaKind.Video); + //Transceiver.LocalVideoTrack = LocalVideoTrack.CreateFromSource(VideoSource, new LocalVideoTrackInitConfig() + //{ + // trackName = "ScreenCapture" + //}); + + PeerSession.CreateOffer(); } + public void SendAudioSample(byte[] audioSample) { SendDto(new AudioSampleDto(audioSample)); @@ -116,6 +137,7 @@ namespace Remotely.Desktop.Core.Services { SendDto(new MachineNameDto(machineName)); } + public void SendScreenData(string selectedScreen, string[] displayNames) { SendDto(new ScreenDataDto(selectedScreen, displayNames)); @@ -125,19 +147,32 @@ namespace Remotely.Desktop.Core.Services { SendDto(new ScreenSizeDto(width, height)); } + public void SendWindowsSessions(List windowsSessions) { SendDto(new WindowsSessionsDto(windowsSessions)); } - public void SetRemoteDescription(string type, string sdp) + public async Task SetRemoteDescription(string type, string sdp) { - PeerConnection.SetRemoteDescription(type, sdp); - if (type == "offer") + if (!Enum.TryParse(type, true, out var sdpMessageType)) { - PeerConnection.CreateAnswer(); + Logger.Write("Unable to parse remote WebRTC description type."); + return; + } + + await PeerSession.SetRemoteDescriptionAsync(new SdpMessage() + { + Content = sdp, + Type = sdpMessageType + }); + + if (sdpMessageType == SdpMessageType.Offer) + { + PeerSession.CreateAnswer(); } } + private async void CaptureChannel_MessageReceived(byte[] obj) { Logger.Debug($"DataChannel message received. Size: {obj.Length}"); @@ -152,31 +187,61 @@ namespace Remotely.Desktop.Core.Services await Init(IceServers); } } + private void DataChannel_BufferingChanged(ulong previous, ulong current, ulong limit) { CurrentBuffer = current; } + //private void GetCaptureFrame(in FrameRequest request) + //{ + // Viewer.Capturer.GetNextFrame(); + // using (var bitmapCopy = (Bitmap)Viewer.Capturer.CurrentFrame.Clone()) + // { + // var bitmapData = bitmapCopy.LockBits( + // Viewer.Capturer.CurrentScreenBounds, + // System.Drawing.Imaging.ImageLockMode.ReadOnly, + // System.Drawing.Imaging.PixelFormat.Format32bppArgb); + + // try + // { + // var frame = new Argb32VideoFrame() + // { + // data = bitmapData.Scan0, + // height = (uint)bitmapCopy.Height, + // width = (uint)bitmapCopy.Width, + // stride = bitmapData.Stride + // }; + // request.CompleteRequest(in frame); + // } + // finally + // { + // bitmapCopy.UnlockBits(bitmapData); + // } + + // } + + //} + private void PeerConnection_Connected() { Logger.Debug("PeerConnection connected."); } - private void PeerConnection_IceCandidateReadytoSend(string candidate, int sdpMlineindex, string sdpMid) - { - Logger.Debug("Ice candidate ready to send."); - IceCandidateReady?.Invoke(this, (candidate, sdpMlineindex, sdpMid)); - } - private void PeerConnection_IceStateChanged(IceConnectionState newState) { Logger.Debug($"Ice state changed to {newState}."); } - private void PeerConnection_LocalSdpReadytoSend(string type, string sdp) + private void PeerSession_IceCandidateReadytoSend(IceCandidate candidate) + { + Logger.Debug("Ice candidate ready to send."); + IceCandidateReady?.Invoke(this, candidate); + } + private void PeerSession_LocalSdpReadytoSend(SdpMessage message) { Logger.Debug($"Local SDP ready."); - LocalSdpReady?.Invoke(this, sdp); + LocalSdpReady?.Invoke(this, message); } private void SendDto(T dto) { diff --git a/Desktop.Core/Services/WebRtcSessionFactory.cs b/Desktop.Core/Services/WebRtcSessionFactory.cs index f503b84d..111e2def 100644 --- a/Desktop.Core/Services/WebRtcSessionFactory.cs +++ b/Desktop.Core/Services/WebRtcSessionFactory.cs @@ -44,7 +44,7 @@ namespace Remotely.Desktop.Core.Services ClipboardService, FileDownloadService); - return new WebRtcSession(messageHandler); + return new WebRtcSession(viewer, messageHandler); } } } diff --git a/Desktop.Linux/Desktop.Linux.csproj b/Desktop.Linux/Desktop.Linux.csproj index 397f3699..bf0d40dd 100644 --- a/Desktop.Linux/Desktop.Linux.csproj +++ b/Desktop.Linux/Desktop.Linux.csproj @@ -39,9 +39,9 @@ - - - + + + diff --git a/Desktop.Win/Desktop.Win.csproj b/Desktop.Win/Desktop.Win.csproj index ebc3a7c0..c5aa1eb2 100644 --- a/Desktop.Win/Desktop.Win.csproj +++ b/Desktop.Win/Desktop.Win.csproj @@ -36,7 +36,7 @@ - + diff --git a/Desktop.Win/Services/ScreenCapturerWin.cs b/Desktop.Win/Services/ScreenCapturerWin.cs index d9d483e6..3f3aeb90 100644 --- a/Desktop.Win/Services/ScreenCapturerWin.cs +++ b/Desktop.Win/Services/ScreenCapturerWin.cs @@ -74,19 +74,14 @@ namespace Remotely.Desktop.Win.Services { try { + Win32Interop.SwitchToInputDesktop(); + if (NeedsInit) { - Logger.Write("Init needed in DXCapture. Switching desktops."); - if (Win32Interop.SwitchToInputDesktop()) - { - Win32Interop.GetCurrentDesktop(out var desktopName); - Logger.Write($"Switch to desktop {desktopName} after capture error in DXCapture."); - } + Logger.Write("Init needed in GetNextFrame."); Init(); } - Win32Interop.SwitchToInputDesktop(); - PreviousFrame?.Dispose(); PreviousFrame = (Bitmap)CurrentFrame.Clone(); diff --git a/Server/Server.csproj b/Server/Server.csproj index 3df91e3c..c63fb090 100644 --- a/Server/Server.csproj +++ b/Server/Server.csproj @@ -86,7 +86,7 @@ - + diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 0612d4a0..06b2bc66 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -15,7 +15,7 @@ - +