diff --git a/.github/workflows/deploy-to-iis.yml b/.github/workflows/deploy-to-iis.yml index 660b79c5..0697b698 100644 --- a/.github/workflows/deploy-to-iis.yml +++ b/.github/workflows/deploy-to-iis.yml @@ -145,11 +145,6 @@ jobs: Write-Host "Setting current version to $CurrentVersion." - # Run the Publish script to build clients and server. - - name: Run Publish script - shell: powershell - run: .\Utilities\Publish.ps1 -CertificatePath "$env:GITHUB_WORKSPACE\GitHubActionsWorkflow.pfx" -CertificatePassword $env:PfxKey -Hostname $env:SiteUrl -CurrentVersion $env:CurrentVersion - # Create MSDeploy Publishing Profile - name: Create MSDeploy Profile shell: powershell @@ -172,7 +167,7 @@ jobs: WMSVC True $env:MsDeployUsername - netcoreapp3.1 + net5.0 true true win-x64 @@ -182,8 +177,19 @@ jobs: New-Item -Path "$env:GITHUB_WORKSPACE\Server\Properties\PublishProfiles\" -ItemType Directory -Force Set-Content -Path "$env:GITHUB_WORKSPACE\Server\Properties\PublishProfiles\DeployIIS.pubxml" -Value $PublishProfile -Force + + # Run the Publish script to build clients and server. + - name: Run Publish script + shell: powershell + run: | + .\Utilities\Publish.ps1 -CertificatePath "$env:GITHUB_WORKSPACE\GitHubActionsWorkflow.pfx" -CertificatePassword $env:PfxKey -Hostname $env:SiteUrl -CurrentVersion $env:CurrentVersion -RID win-x64 -OutDir "$env:GITHUB_WORKSPACE\publish" + + # Upload build artifact to be deployed from Ubuntu runner + - name: Upload build artifact + uses: actions/upload-artifact@v2 + with: + path: ./publish/ - # Uncomment the below to enable deployment to the server. # Publish server to IIS - name: Publish run: | diff --git a/Desktop.Core/Models/CaptureFrame.cs b/Desktop.Core/Models/CaptureFrame.cs new file mode 100644 index 00000000..d5d45a61 --- /dev/null +++ b/Desktop.Core/Models/CaptureFrame.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Remotely.Desktop.Core.Models +{ + public class CaptureFrame + { + public byte[] EncodedImageBytes { get; init; } + public int Top { get; init; } + public int Left { get; init; } + public int Height { get; init; } + public int Width { get; init; } + } +} diff --git a/Desktop.Core/Services/ScreenCaster.cs b/Desktop.Core/Services/ScreenCaster.cs index f1e7d981..53aa9ea5 100644 --- a/Desktop.Core/Services/ScreenCaster.cs +++ b/Desktop.Core/Services/ScreenCaster.cs @@ -1,10 +1,13 @@ using Microsoft.Extensions.DependencyInjection; using Remotely.Desktop.Core.Enums; using Remotely.Desktop.Core.Interfaces; +using Remotely.Desktop.Core.Models; using Remotely.Desktop.Core.Utilities; +using Remotely.Shared.Helpers; using Remotely.Shared.Models; using Remotely.Shared.Utilities; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Drawing; @@ -87,20 +90,29 @@ namespace Remotely.Desktop.Core.Services { if (initialFrame != null) { - await viewer.SendScreenCapture( - ImageUtils.EncodeBitmap(initialFrame, viewer.EncoderParams), - viewer.Capturer.CurrentScreenBounds.Left, - viewer.Capturer.CurrentScreenBounds.Top, - viewer.Capturer.CurrentScreenBounds.Width, - viewer.Capturer.CurrentScreenBounds.Height); + await viewer.SendScreenCapture(new CaptureFrame[] + { + new CaptureFrame() + { + EncodedImageBytes = ImageUtils.EncodeBitmap(initialFrame, viewer.EncoderParams), + Left = viewer.Capturer.CurrentScreenBounds.Left, + Top = viewer.Capturer.CurrentScreenBounds.Top, + Width = viewer.Capturer.CurrentScreenBounds.Width, + Height = viewer.Capturer.CurrentScreenBounds.Height + } + }); } } + if (EnvironmentHelper.IsWindows) { await viewer.InitializeWebRtc(); } + // Wait until the first image is received. + TaskHelper.DelayUntil(() => !viewer.PendingSentFrames.Any(), TimeSpan.MaxValue); + while (!viewer.DisconnectRequested && viewer.IsConnected) { try @@ -137,15 +149,20 @@ namespace Remotely.Desktop.Core.Services currentFrame?.Dispose(); currentFrame = viewer.Capturer.GetNextFrame(); - var diffAreas = ImageUtils.GetDiffAreas(currentFrame, previousFrame, viewer.Capturer.CaptureFullscreen); + var diffAreas = ImageUtils.GetDiffAreas2(currentFrame, previousFrame, viewer.Capturer.CaptureFullscreen); - if (diffAreas.Count == 0) + if (!diffAreas.Any()) { continue; } + + viewer.Capturer.CaptureFullscreen = false; + + var frameClone = (Bitmap)currentFrame.Clone(); + Debug.WriteLine($"Sending {diffAreas.Count} frames."); await sendFramesLock.WaitAsync(); - SendFrames((Bitmap)currentFrame.Clone(), diffAreas, viewer, sendFramesLock); + SendFrames(frameClone, diffAreas, viewer, sendFramesLock); } catch (Exception ex) { @@ -172,28 +189,65 @@ namespace Remotely.Desktop.Core.Services } } - private void SendFrames(Bitmap currentFrame, List diffAreas, Viewer viewer, SemaphoreSlim sendFramesLock) + private void SendFrame(Bitmap diffImage, Viewer viewer, SemaphoreSlim sendFramesLock) { _ = Task.Run(async () => { try { + var encodedImageBytes = ImageUtils.EncodeGif(diffImage); + + if (encodedImageBytes?.Length > 0) + { + var frames = new List() + { + new CaptureFrame() + { + EncodedImageBytes = encodedImageBytes, + Top = 0, + Left = 0, + Width = diffImage.Width, + Height = diffImage.Height, + } + }; + await viewer.SendScreenCapture(frames); + } + } + finally + { + sendFramesLock.Release(); + diffImage.Dispose(); + } + }); + } + + private static void SendFrames(Bitmap currentFrame, ICollection diffAreas, Viewer viewer, SemaphoreSlim sendFramesLock) + { + _ = Task.Run(async () => + { + try + { + var frames = new List(); + foreach (var diffArea in diffAreas) { using var newImage = currentFrame.Clone(diffArea, PixelFormat.Format32bppArgb); - if (viewer.Capturer.CaptureFullscreen) - { - viewer.Capturer.CaptureFullscreen = false; - } - var encodedImageBytes = ImageUtils.EncodeBitmap(newImage, viewer.EncoderParams); if (encodedImageBytes?.Length > 0) { - await viewer.SendScreenCapture(encodedImageBytes, diffArea.Left, diffArea.Top, diffArea.Width, diffArea.Height); + frames.Add(new CaptureFrame() + { + EncodedImageBytes = encodedImageBytes, + Top = diffArea.Top, + Left = diffArea.Left, + Width = diffArea.Width, + Height = diffArea.Height, + }); } - } + }; + await viewer.SendScreenCapture(frames); } finally { diff --git a/Desktop.Core/Services/Viewer.cs b/Desktop.Core/Services/Viewer.cs index ca59a188..b80e0fff 100644 --- a/Desktop.Core/Services/Viewer.cs +++ b/Desktop.Core/Services/Viewer.cs @@ -1,4 +1,5 @@ using Remotely.Desktop.Core.Interfaces; +using Remotely.Desktop.Core.Models; using Remotely.Desktop.Core.ViewModels; using Remotely.Shared.Helpers; using Remotely.Shared.Models; @@ -6,7 +7,9 @@ using Remotely.Shared.Models.RemoteControlDtos; using Remotely.Shared.Utilities; using Remotely.Shared.Win32; using System; +using System.Collections; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Drawing.Imaging; using System.IO; using System.Linq; @@ -17,7 +20,7 @@ namespace Remotely.Desktop.Core.Services public class Viewer : IDisposable { private readonly int _defaultImageQuality = 60; - private int _imageQuality; + private long _imageQuality; private DateTimeOffset _lastQualityAdjustment; public Viewer(ICasterSocket casterSocket, IScreenCapturer screenCapturer, @@ -43,7 +46,7 @@ namespace Remotely.Desktop.Core.Services public bool DisconnectRequested { get; set; } public EncoderParameters EncoderParams { get; private set; } public bool HasControl { get; set; } = true; - public int ImageQuality + public long ImageQuality { get { @@ -247,39 +250,63 @@ namespace Remotely.Desktop.Core.Services () => CasterSocket.SendDtoToViewer(dto, ViewerConnectionID)); } - public async Task SendScreenCapture(byte[] encodedImageBytes, int left, int top, int width, int height) + public async Task SendScreenCapture(IEnumerable screenFrame) { PendingSentFrames.Enqueue(DateTimeOffset.Now); - for (var i = 0; i < encodedImageBytes.Length; i += 50_000) + foreach (var frame in screenFrame) { - var dto = new CaptureFrameDto() + var left = frame.Left; + var top = frame.Top; + var width = frame.Width; + var height = frame.Height; + + for (var i = 0; i < frame.EncodedImageBytes.Length; i += 50_000) + { + var dto = new CaptureFrameDto() + { + Left = left, + Top = top, + Width = width, + Height = height, + EndOfFrame = false, + ImageBytes = frame.EncodedImageBytes.Skip(i).Take(50_000).ToArray(), + ImageQuality = _imageQuality, + EndOfCapture = false + }; + + await SendToViewer(() => RtcSession.SendDto(dto), + () => CasterSocket.SendDtoToViewer(dto, ViewerConnectionID)); + } + + var endOfFrameDto = new CaptureFrameDto() { Left = left, Top = top, Width = width, Height = height, - EndOfFrame = false, - ImageBytes = encodedImageBytes.Skip(i).Take(50_000).ToArray(), - ImageQuality = _imageQuality + EndOfFrame = true, + ImageQuality = _imageQuality, + EndOfCapture = false }; - await SendToViewer(() => RtcSession.SendDto(dto), - () => CasterSocket.SendDtoToViewer(dto, ViewerConnectionID)); + await SendToViewer(() => RtcSession.SendDto(endOfFrameDto), + () => CasterSocket.SendDtoToViewer(endOfFrameDto, ViewerConnectionID)); } - var endOfFrameDto = new CaptureFrameDto() + var endofCaptureDto = new CaptureFrameDto() { - Left = left, - Top = top, - Width = width, - Height = height, + Left = 0, + Top = 0, + Width = 0, + Height = 0, EndOfFrame = true, - ImageQuality = _imageQuality + ImageQuality = _imageQuality, + EndOfCapture = true }; - await SendToViewer(() => RtcSession.SendDto(endOfFrameDto), - () => CasterSocket.SendDtoToViewer(endOfFrameDto, ViewerConnectionID)); + await SendToViewer(() => RtcSession.SendDto(endofCaptureDto), + () => CasterSocket.SendDtoToViewer(endofCaptureDto, ViewerConnectionID)); } public async Task SendScreenData(string selectedScreen, string[] displayNames) diff --git a/Desktop.Core/Utilities/ImageUtils.cs b/Desktop.Core/Utilities/ImageUtils.cs index 63a5f59d..f93a151d 100644 --- a/Desktop.Core/Utilities/ImageUtils.cs +++ b/Desktop.Core/Utilities/ImageUtils.cs @@ -1,23 +1,36 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Drawing; using System.Drawing.Imaging; using System.IO; using System.Linq; using System.Runtime.InteropServices; +using System.Threading.Tasks; namespace Remotely.Desktop.Core.Utilities { public class ImageUtils { public static ImageCodecInfo JpegEncoder { get; } = ImageCodecInfo.GetImageEncoders().FirstOrDefault(x => x.FormatID == ImageFormat.Jpeg.Guid); + public static ImageCodecInfo GifEncoder { get; } = ImageCodecInfo.GetImageEncoders().FirstOrDefault(x => x.FormatID == ImageFormat.Gif.Guid); + public static byte[] EncodeBitmap(Bitmap bitmap, EncoderParameters encoderParams) { + using var ms = new MemoryStream(); bitmap.Save(ms, JpegEncoder, encoderParams); return ms.ToArray(); } + public static byte[] EncodeGif(Bitmap diffImage) + { + diffImage.MakeTransparent(Color.FromArgb(0, 0, 0, 0)); + using var ms = new MemoryStream(); + diffImage.Save(ms, ImageFormat.Gif); + return ms.ToArray(); + } + public static List GetDiffAreas(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen) { var changes = new List(); @@ -106,7 +119,7 @@ namespace Remotely.Desktop.Core.Utilities } if (!changeOnCurrentRow && changeOnPreviousRow && - left <= right && + left <= right && top <= bottom) { AddChangeToList(changes, left, top, right, bottom, width, height); @@ -138,10 +151,267 @@ namespace Remotely.Desktop.Core.Utilities } } - public static Bitmap GetImageDiff(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen) + public static ICollection GetDiffAreas2(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen) { + if (currentFrame == null || previousFrame == null) + { + return Array.Empty(); + } + if (captureFullscreen) { + return new Rectangle[] { new Rectangle(new Point(0, 0), currentFrame.Size) }; + } + + if (currentFrame.Height != previousFrame.Height || currentFrame.Width != previousFrame.Width) + { + throw new Exception("Bitmaps are not of equal dimensions."); + } + if (currentFrame.PixelFormat != previousFrame.PixelFormat) + { + throw new Exception("Bitmaps are not the same format."); + } + + var width = currentFrame.Width; + var height = currentFrame.Height; + + BitmapData bd1 = null; + BitmapData bd2 = null; + + var bytesPerPixel = Bitmap.GetPixelFormatSize(currentFrame.PixelFormat) / 8; + var numberOfPixels = width * height; + var totalSize = numberOfPixels * bytesPerPixel; + var changes = new ConcurrentQueue(); + try + { + bd1 = previousFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, currentFrame.PixelFormat); + bd2 = currentFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, previousFrame.PixelFormat); + + unsafe + { + + byte* scan1 = (byte*)bd1.Scan0.ToPointer(); + byte* scan2 = (byte*)bd2.Scan0.ToPointer(); + + + var gridColumnWidth = width % 8 == 0 ? width / 8 : + width % 2 == 0 ? width / 2 : + width; + + + var gridRowHeight = height % 9 == 0 ? height / 9 : + height % 10 == 0 ? height / 10 : + height % 4 == 0 ? height / 4 : + height; + + var gridColumns = Enumerable.Range(0, width).Where(i => i % gridColumnWidth == 0); + var gridRows = Enumerable.Range(0, height).Where(i => i % gridRowHeight == 0); + + + Parallel.ForEach(gridColumns, gridColumn => + { + Parallel.ForEach(gridRows, gridRow => + { + int left = int.MaxValue; + int top = int.MaxValue; + int right = int.MinValue; + int bottom = int.MinValue; + + for (var row = 0; row < gridRowHeight; row++) + { + for (var col = 0; col < gridColumnWidth; col++) + { + var pixelLeft = gridColumn + col; + var pixelTop = gridRow + row; + + + var rowIndex = pixelTop * width * bytesPerPixel; + + var columnIndex = pixelLeft * bytesPerPixel; + + var i = rowIndex + columnIndex; + + byte* data1 = scan1 + i; + byte* data2 = scan2 + i; + + if (data1[0] != data2[0] || + data1[1] != data2[1] || + data1[2] != data2[2] || + data1[3] != data2[3]) + { + + if (pixelTop < top) + { + top = pixelTop; + } + if (pixelTop > bottom) + { + bottom = pixelTop; + } + if (pixelLeft < left) + { + left = pixelLeft; + } + if (pixelLeft > right) + { + right = pixelLeft; + } + } + } + } + + if (left <= right && top <= bottom) + { + AddChangeToList(changes, left, top, right, bottom, width, height); + } + }); + }); + + return changes.ToArray(); + } + } + catch + { + return changes.ToArray(); + } + finally + { + currentFrame.UnlockBits(bd1); + previousFrame.UnlockBits(bd2); + } + } + + public static ICollection GetDiffAreas3(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen) + { + if (currentFrame == null || previousFrame == null) + { + return Array.Empty(); + } + + if (captureFullscreen) + { + return new Rectangle[] { new Rectangle(new Point(0, 0), currentFrame.Size) }; + } + + if (currentFrame.Height != previousFrame.Height || currentFrame.Width != previousFrame.Width) + { + throw new Exception("Bitmaps are not of equal dimensions."); + } + if (currentFrame.PixelFormat != previousFrame.PixelFormat) + { + throw new Exception("Bitmaps are not the same format."); + } + + var width = currentFrame.Width; + var height = currentFrame.Height; + + BitmapData bd1 = null; + BitmapData bd2 = null; + + var bytesPerPixel = Bitmap.GetPixelFormatSize(currentFrame.PixelFormat) / 8; + var numberOfPixels = width * height; + var totalSize = numberOfPixels * bytesPerPixel; + var changes = new ConcurrentQueue(); + try + { + bd1 = previousFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, currentFrame.PixelFormat); + bd2 = currentFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, previousFrame.PixelFormat); + + unsafe + { + + byte* scan1 = (byte*)bd1.Scan0.ToPointer(); + byte* scan2 = (byte*)bd2.Scan0.ToPointer(); + + var gridRowHeight = height % 9 == 0 ? height / 9 : + height % 10 == 0 ? height / 10 : + height % 4 == 0 ? height / 4 : + height; + + var gridRows = Enumerable.Range(0, height).Where(i => i % gridRowHeight == 0); + + Parallel.ForEach(gridRows, gridRow => + { + int left = int.MaxValue; + int top = int.MaxValue; + int right = int.MinValue; + int bottom = int.MinValue; + + for (var row = 0; row < gridRowHeight; row++) + { + for (var col = 0; col < width; col++) + { + var pixelLeft = col; + var pixelTop = gridRow + row; + + + var rowIndex = pixelTop * width * bytesPerPixel; + + var columnIndex = pixelLeft * bytesPerPixel; + + var i = rowIndex + columnIndex; + + byte* data1 = scan1 + i; + byte* data2 = scan2 + i; + + if (data1[0] != data2[0] || + data1[1] != data2[1] || + data1[2] != data2[2] || + data1[3] != data2[3]) + { + + if (pixelTop < top) + { + top = pixelTop; + } + if (pixelTop > bottom) + { + bottom = pixelTop; + } + if (pixelLeft < left) + { + left = pixelLeft; + } + if (pixelLeft > right) + { + right = pixelLeft; + } + } + } + } + if (left <= right && top <= bottom) + { + AddChangeToList(changes, left, top, right, bottom, width, height); + } + }); + + return changes.ToArray(); + } + } + catch + { + return changes.ToArray(); + } + finally + { + currentFrame.UnlockBits(bd1); + previousFrame.UnlockBits(bd2); + } + } + + + + public static Bitmap GetImageDiff(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen, out bool hadChanges) + { + hadChanges = false; + if (currentFrame is null || previousFrame is null) + { + hadChanges = false; + return null; + } + if (captureFullscreen) + { + hadChanges = true; return (Bitmap)currentFrame.Clone(); } @@ -185,6 +455,7 @@ namespace Remotely.Desktop.Core.Utilities data1[2] != data2[2] || data1[3] != data2[3]) { + hadChanges = true; data3[0] = data2[0]; data3[1] = data2[1]; data3[2] = data2[2]; @@ -213,12 +484,24 @@ namespace Remotely.Desktop.Core.Utilities { // Bounding box is valid. Padding is necessary to prevent artifacts from // moving windows. - left = Math.Max(left - 5, 0); - top = Math.Max(top - 5, 0); - right = Math.Min(right + 5, width); - bottom = Math.Min(bottom + 5, height); + left = Math.Max(left - 1, 0); + top = Math.Max(top - 1, 0); + right = Math.Min(right + 1, width); + bottom = Math.Min(bottom + 1, height); changes.Add(new Rectangle(left, top, right - left, bottom - top)); } + + private static void AddChangeToList(ConcurrentQueue changes, int left, int top, int right, int bottom, int width, int height) + { + // Bounding box is valid. Padding is necessary to prevent artifacts from + // moving windows. + left = Math.Max(left - 1, 0); + top = Math.Max(top - 1, 0); + right = Math.Min(right + 1, width); + bottom = Math.Min(bottom + 1, height); + + changes.Enqueue(new Rectangle(left, top, right - left, bottom - top)); + } } } diff --git a/Desktop.Win/Program.cs b/Desktop.Win/Program.cs index f6b9f436..685df362 100644 --- a/Desktop.Win/Program.cs +++ b/Desktop.Win/Program.cs @@ -35,7 +35,7 @@ namespace Remotely.Desktop.Win } } } - [STAThread] + public static void Main(string[] args) { try @@ -89,19 +89,6 @@ namespace Remotely.Desktop.Win } } - private static void WaitForAppExit() - { - var appExitEvent = new ManualResetEventSlim(); - App.Current.Dispatcher.Invoke(() => - { - App.Current.Exit += (s, a) => - { - appExitEvent.Set(); - }; - }); - appExitEvent.Wait(); - } - private static void BuildServices() { var serviceCollection = new ServiceCollection(); @@ -165,6 +152,7 @@ namespace Remotely.Desktop.Win private static async Task StartScreenCasting() { + CursorIconWatcher = Services.GetRequiredService(); await CasterSocket.Connect(Conductor.Host); @@ -226,5 +214,18 @@ namespace Remotely.Desktop.Win } Logger.Write("Background UI apps started."); } + + private static void WaitForAppExit() + { + var appExitEvent = new ManualResetEventSlim(); + App.Current.Dispatcher.Invoke(() => + { + App.Current.Exit += (s, a) => + { + appExitEvent.Set(); + }; + }); + appExitEvent.Wait(); + } } } diff --git a/Desktop.Win/ViewModels/MainWindowViewModel.cs b/Desktop.Win/ViewModels/MainWindowViewModel.cs index a0c1d11d..0e56f20c 100644 --- a/Desktop.Win/ViewModels/MainWindowViewModel.cs +++ b/Desktop.Win/ViewModels/MainWindowViewModel.cs @@ -25,8 +25,6 @@ namespace Remotely.Desktop.Win.ViewModels private string _host; private string _sessionID; - public static MainWindowViewModel Current { get; private set; } - public MainWindowViewModel() { Current = this; @@ -50,6 +48,7 @@ namespace Remotely.Desktop.Win.ViewModels Conductor.ScreenCastRequested += ScreenCastRequested; } + public static MainWindowViewModel Current { get; private set; } public static IServiceProvider Services => ServiceContainer.Instance; public ICommand ChangeServerCommand @@ -64,10 +63,6 @@ namespace Remotely.Desktop.Win.ViewModels } } - private Conductor Conductor { get; set; } - private ICasterSocket CasterSocket { get; set; } - private ICursorIconWatcher CursorIconWatcher { get; set; } - public ICommand ElevateToAdminCommand { get @@ -178,6 +173,12 @@ namespace Remotely.Desktop.Win.ViewModels public ObservableCollection Viewers { get; } = new ObservableCollection(); + private ICasterSocket CasterSocket { get; set; } + + private Conductor Conductor { get; set; } + + private ICursorIconWatcher CursorIconWatcher { get; set; } + public void CopyLink() { Clipboard.SetText($"{Host}/RemoteControl?sessionID={SessionID?.Replace(" ", "")}"); @@ -192,7 +193,7 @@ namespace Remotely.Desktop.Win.ViewModels public async Task Init() { SessionID = "Retrieving..."; - + Host = Config.GetConfig().Host; while (string.IsNullOrWhiteSpace(Host)) @@ -263,6 +264,10 @@ namespace Remotely.Desktop.Win.ViewModels } } + public void ShutdownApp() + { + Services.GetRequiredService().Shutdown(); + } private void Application_Exit(object sender, ExitEventArgs e) { App.Current.Dispatcher.Invoke(() => diff --git a/Desktop.Win/Views/MainWindow.xaml b/Desktop.Win/Views/MainWindow.xaml index 9c8a56ea..98bcfc3a 100644 --- a/Desktop.Win/Views/MainWindow.xaml +++ b/Desktop.Win/Views/MainWindow.xaml @@ -10,6 +10,7 @@ WindowStyle="None" ResizeMode="NoResize" MouseLeftButtonDown="Window_MouseLeftButtonDown" + Closing="Window_Closing" Loaded="Window_Loaded" Icon="/Assets/Remotely_Icon.png"> diff --git a/Desktop.Win/Views/MainWindow.xaml.cs b/Desktop.Win/Views/MainWindow.xaml.cs index 29b1f4e2..a237971f 100644 --- a/Desktop.Win/Views/MainWindow.xaml.cs +++ b/Desktop.Win/Views/MainWindow.xaml.cs @@ -47,7 +47,7 @@ namespace Remotely.Desktop.Win.Views private void MinimizeButton_Click(object sender, RoutedEventArgs e) { - this.WindowState = WindowState.Minimized; + WindowState = WindowState.Minimized; } private void OptionsButton_Click(object sender, RoutedEventArgs e) @@ -55,6 +55,11 @@ namespace Remotely.Desktop.Win.Views (sender as Button).ContextMenu.IsOpen = true; } + private void Window_Closing(object sender, CancelEventArgs e) + { + ViewModel?.ShutdownApp(); + } + private async void Window_Loaded(object sender, RoutedEventArgs e) { if (!DesignerProperties.GetIsInDesignMode(this) && diff --git a/Server/wwwroot/src/RemoteControl/DtoMessageHandler.ts b/Server/wwwroot/src/RemoteControl/DtoMessageHandler.ts index 83d0e666..f0859646 100644 --- a/Server/wwwroot/src/RemoteControl/DtoMessageHandler.ts +++ b/Server/wwwroot/src/RemoteControl/DtoMessageHandler.ts @@ -17,18 +17,17 @@ import { } from "./Interfaces/Dtos.js"; import { ReceiveFile } from "./FileTransferService.js"; - export class DtoMessageHandler { MessagePack: any = window['MessagePack']; - PartialCaptureFrames: Uint8Array[] = []; - ParseBinaryMessage(data: ArrayBuffer) { + PartialCaptures: Record = {}; + async ParseBinaryMessage(data: ArrayBuffer) { var model = this.MessagePack.decode(data) as BaseDto; switch (model.DtoType) { case BaseDtoType.AudioSample: this.HandleAudioSample(model as unknown as AudioSampleDto); break; case BaseDtoType.CaptureFrame: - this.HandleCaptureFrame(model as unknown as CaptureFrameDto); + await this.HandleCaptureFrame(model as unknown as CaptureFrameDto); break; case BaseDtoType.ClipboardText: this.HandleClipboardText(model as unknown as ClipboardTextDto); @@ -58,31 +57,72 @@ export class DtoMessageHandler { HandleAudioSample(audioSample: AudioSampleDto) { Sound.Play(audioSample.Buffer); } - HandleCaptureFrame(captureFrame: CaptureFrameDto) { + + async HandleCaptureFrame(captureFrame: CaptureFrameDto) { if (UI.AutoQualityAdjustCheckBox.checked && Number(UI.QualitySlider.value) != captureFrame.ImageQuality) { UI.QualitySlider.value = String(captureFrame.ImageQuality); } - if (captureFrame.EndOfFrame) { + if (captureFrame.EndOfCapture) { ViewerApp.MessageSender.SendFrameReceived(); - var url = window.URL.createObjectURL(new Blob(this.PartialCaptureFrames)); - var img = document.createElement("img"); - img.onload = () => { - UI.Screen2DContext.drawImage(img, - captureFrame.Left, - captureFrame.Top, - captureFrame.Width, - captureFrame.Height); - window.URL.revokeObjectURL(url); - }; - img.src = url; - this.PartialCaptureFrames = []; + + Object.keys(this.PartialCaptures).forEach(async x => { + let partial = this.PartialCaptures[x]; + let firstFrame = partial[0]; + let frameBytes = partial.map(x => x.ImageBytes); + + let bitmap = await createImageBitmap(new Blob(frameBytes)); + + UI.Screen2DContext.drawImage(bitmap, + firstFrame.Left, + firstFrame.Top, + firstFrame.Width, + firstFrame.Height); + + bitmap.close(); + }) + + this.PartialCaptures = {}; } + //else if (captureFrame.EndOfFrame) { + // let key = `${captureFrame.Left},${captureFrame.Top}`; + // let frameBytes = this.PartialCaptures[key].map(x => x.ImageBytes); + + // //var url = window.URL.createObjectURL(new Blob(frameBytes)); + // //var img = document.createElement("img"); + // //img.onload = () => { + // // UI.StagingRenderer.drawImage(img, + // // captureFrame.Left, + // // captureFrame.Top, + // // captureFrame.Width, + // // captureFrame.Height); + // // window.URL.revokeObjectURL(url); + // //}; + // //img.src = url; + + + // let bitmap = await createImageBitmap(new Blob(frameBytes)); + + // UI.StagingRenderer.drawImage(bitmap, + // captureFrame.Left, + // captureFrame.Top, + // captureFrame.Width, + // captureFrame.Height); + + // bitmap.close(); + //} else { - this.PartialCaptureFrames.push(captureFrame.ImageBytes); + let key = `${captureFrame.Left},${captureFrame.Top}`; + if (this.PartialCaptures[key]) { + this.PartialCaptures[key].push(captureFrame); + } + else { + this.PartialCaptures[key] = [captureFrame]; + } } } + HandleClipboardText(clipboardText: ClipboardTextDto) { ViewerApp.ClipboardWatcher.SetClipboardText(clipboardText.ClipboardText); ShowMessage("Clipboard updated."); diff --git a/Server/wwwroot/src/RemoteControl/InputEventHandlers.ts b/Server/wwwroot/src/RemoteControl/InputEventHandlers.ts index 71120cfb..1b5481ce 100644 --- a/Server/wwwroot/src/RemoteControl/InputEventHandlers.ts +++ b/Server/wwwroot/src/RemoteControl/InputEventHandlers.ts @@ -567,10 +567,12 @@ export function ApplyInputHandlers() { if (document.querySelector("input:focus") || document.querySelector("textarea:focus")) { return; } - e.preventDefault(); if (ViewerApp.ViewOnlyMode) { return; } + if (!e.ctrlKey || !e.shiftKey || e.key.toLowerCase() != "i") { + e.preventDefault(); + } ViewerApp.MessageSender.SendKeyDown(e.key); }); window.addEventListener("keyup", function (e) { diff --git a/Server/wwwroot/src/RemoteControl/Interfaces/Dtos.ts b/Server/wwwroot/src/RemoteControl/Interfaces/Dtos.ts index ff6a9553..5e84db1c 100644 --- a/Server/wwwroot/src/RemoteControl/Interfaces/Dtos.ts +++ b/Server/wwwroot/src/RemoteControl/Interfaces/Dtos.ts @@ -18,6 +18,7 @@ export interface AudioSampleDto extends BaseDto { export interface CaptureFrameDto extends BaseDto { EndOfFrame: boolean; + EndOfCapture: boolean; Left: number; Top: number; Width: number; diff --git a/Server/wwwroot/src/RemoteControl/RtcSession.ts b/Server/wwwroot/src/RemoteControl/RtcSession.ts index ecf37639..b55d09c4 100644 --- a/Server/wwwroot/src/RemoteControl/RtcSession.ts +++ b/Server/wwwroot/src/RemoteControl/RtcSession.ts @@ -46,7 +46,7 @@ export class RtcSession { }; this.DataChannel.onmessage = async (ev) => { var data = ev.data as ArrayBuffer; - ViewerApp.DtoMessageHandler.ParseBinaryMessage(data); + await ViewerApp.DtoMessageHandler.ParseBinaryMessage(data); }; this.DataChannel.onopen = (ev) => { diff --git a/Server/wwwroot/src/RemoteControl/UI.ts b/Server/wwwroot/src/RemoteControl/UI.ts index 5a009861..a4af1b63 100644 --- a/Server/wwwroot/src/RemoteControl/UI.ts +++ b/Server/wwwroot/src/RemoteControl/UI.ts @@ -3,6 +3,9 @@ import { ConvertUInt8ArrayToBase64 } from "../Shared/Utilities.js"; import { WindowsSession } from "../Shared/Models/WindowsSession.js"; import { WindowsSessionType } from "../Shared/Enums/WindowsSessionType.js"; +const offscreenCanvas = document.createElement("canvas"); +var renderId: number; + export var AudioButton = document.getElementById("audioButton") as HTMLButtonElement; export var MenuButton = document.getElementById("menuButton") as HTMLButtonElement; export var MenuFrame = document.getElementById("menuFrame") as HTMLDivElement; @@ -49,6 +52,8 @@ export var RecordSessionButton = document.getElementById("recordSessionButton") export var DownloadRecordingButton = document.getElementById("downloadRecordingButton") as HTMLButtonElement; export var ViewOnlyButton = document.getElementById("viewOnlyButton") as HTMLButtonElement; export var FullScreenButton = document.getElementById("fullScreenButton") as HTMLButtonElement; +export var StagingCanvas = offscreenCanvas as HTMLCanvasElement; +export var StagingRenderer = offscreenCanvas.getContext("2d"); export function GetCurrentViewer(): HTMLElement { if (ScreenViewer.hasAttribute("hidden")) { @@ -96,15 +101,20 @@ export function Prompt(promptMessage: string): Promise { }); } + export function SetScreenSize(width: number, height: number) { ScreenViewer.width = width; ScreenViewer.height = height; + StagingCanvas.width = width; + StagingCanvas.height = height; Screen2DContext.clearRect(0, 0, width, height); + StagingRenderer.clearRect(0, 0, width, height); } export function ToggleConnectUI(shown: boolean) { if (shown) { Screen2DContext.clearRect(0, 0, ScreenViewer.width, ScreenViewer.height); + StagingRenderer.clearRect(0, 0, ScreenViewer.width, ScreenViewer.height); ScreenViewer.setAttribute("hidden", "hidden"); VideoScreenViewer.setAttribute("hidden", "hidden"); ConnectBox.style.removeProperty("display"); diff --git a/Server/wwwroot/src/RemoteControl/ViewerHubConnection.ts b/Server/wwwroot/src/RemoteControl/ViewerHubConnection.ts index fc6d82ae..5d001c45 100644 --- a/Server/wwwroot/src/RemoteControl/ViewerHubConnection.ts +++ b/Server/wwwroot/src/RemoteControl/ViewerHubConnection.ts @@ -73,8 +73,8 @@ export class ViewerHubConnection { private ApplyMessageHandlers(hubConnection) { - hubConnection.on("SendDtoToBrowser", (dto: ArrayBuffer) => { - ViewerApp.DtoMessageHandler.ParseBinaryMessage(dto); + hubConnection.on("SendDtoToBrowser", async (dto: ArrayBuffer) => { + await ViewerApp.DtoMessageHandler.ParseBinaryMessage(dto); }); hubConnection.on("ClipboardTextChanged", (clipboardText: string) => { ViewerApp.ClipboardWatcher.SetClipboardText(clipboardText); @@ -86,33 +86,7 @@ export class ViewerHubConnection { hubConnection.on("ScreenSize", (width: number, height: number) => { UI.SetScreenSize(width, height); }); - hubConnection.on("ScreenCapture", (buffer: Uint8Array, - left: number, - top: number, - width: number, - height: number, - imageQuality: number, - endOfFrame: boolean) => { - if (UI.AutoQualityAdjustCheckBox.checked && Number(UI.QualitySlider.value) != imageQuality) { - UI.QualitySlider.value = String(imageQuality); - } - - if (endOfFrame) { - this.SendDtoToClient(new GenericDto(BaseDtoType.FrameReceived)); - var url = window.URL.createObjectURL(new Blob(this.PartialCaptureFrames)); - var img = document.createElement("img"); - img.onload = () => { - UI.Screen2DContext.drawImage(img, left, top, width, height); - window.URL.revokeObjectURL(url); - }; - img.src = url; - this.PartialCaptureFrames = []; - } - else { - this.PartialCaptureFrames.push(buffer); - } - }); hubConnection.on("ConnectionFailed", () => { UI.ConnectButton.removeAttribute("disabled"); UI.StatusMessage.innerHTML = "Connection failed or was denied."; diff --git a/Shared/Models/RemoteControlDtos/CaptureFrameDto.cs b/Shared/Models/RemoteControlDtos/CaptureFrameDto.cs index a80d88d6..c60cfcd4 100644 --- a/Shared/Models/RemoteControlDtos/CaptureFrameDto.cs +++ b/Shared/Models/RemoteControlDtos/CaptureFrameDto.cs @@ -9,6 +9,9 @@ namespace Remotely.Shared.Models.RemoteControlDtos [DataMember(Name = "DtoType")] public new BaseDtoType DtoType { get; } = BaseDtoType.CaptureFrame; + [DataMember(Name = "EndOfCapture")] + public bool EndOfCapture { get; set; } + [DataMember(Name = "EndOfFrame")] public bool EndOfFrame { get; set; } diff --git a/Tests/ManualTests.cs b/Tests/ManualTests.cs index bd0667e0..b19eaa96 100644 --- a/Tests/ManualTests.cs +++ b/Tests/ManualTests.cs @@ -5,17 +5,21 @@ using Moq; using Remotely.Desktop.Core; using Remotely.Desktop.Core.Interfaces; using Remotely.Desktop.Core.Services; +using Remotely.Desktop.Core.Utilities; using Remotely.Desktop.Win.Services; using Remotely.Shared.Models; using Remotely.Shared.Models.RemoteControlDtos; -using Remotely.Shared.Utilities; using System; -using System.Collections.Generic; using System.Diagnostics; +using System.Drawing; +using System.Drawing.Imaging; +using System.IO; +using System.IO.Compression; using System.Linq; -using System.Text; +using System.Reflection; using System.Threading; using System.Threading.Tasks; +using System.Windows.Media.Imaging; namespace Remotely.Tests { @@ -52,12 +56,10 @@ namespace Remotely.Tests ViewerID = "asdf" }; - var timeout = Debugger.IsAttached ? - 20_000 : - 5_000; - _ = Task.Run(async () => await _screenCaster.BeginScreenCasting(request)); + var timeout = 5_000; + await Task.Delay(timeout); _viewer.DisconnectRequested = true; @@ -68,6 +70,172 @@ namespace Remotely.Tests _viewer.Dispose(); } + [TestMethod] +#if !DEBUG + [Ignore("Manual test.")] +#endif + public void EncodingTests() + { + for (var i = 0; i < 2; i++) + { + var encoderParams = new EncoderParameters() + { + Param = new EncoderParameter[] + { + new EncoderParameter(Encoder.Quality, 60L) + + } + }; + + using var frame1 = GetFrame("Frame1"); + using var frame2 = GetFrame("Frame2"); + + var sw = Stopwatch.StartNew(); + + + sw.Restart(); + var diffs = ImageUtils.GetDiffAreas(frame1, frame2, false); + Debug.WriteLine($"Diff time: {sw.Elapsed.TotalMilliseconds}"); + + var diffSize = 0; + foreach (var x in diffs) + { + using (var tempImage = (Bitmap)frame1.Clone(new Rectangle(x.X, x.Y, x.Width, x.Height), PixelFormat.Format32bppArgb)) + { + using (var ms = new MemoryStream()) + { + tempImage.Save(ms, ImageFormat.Jpeg); + diffSize += ms.ToArray().Length; + } + } + } + Debug.WriteLine($"Diff size: {diffSize}"); + + + sw.Restart(); + var diff2Size = 0; + var diffs2 = ImageUtils.GetDiffAreas2(frame1, frame2, false); + Debug.WriteLine($"Diff2 time: {sw.Elapsed.TotalMilliseconds}"); + foreach (var x in diffs2) + { + using (var tempImage = (Bitmap)frame1.Clone(new Rectangle(x.X, x.Y, x.Width, x.Height), PixelFormat.Format32bppArgb)) + { + using (var ms = new MemoryStream()) + { + tempImage.Save(ms, ImageFormat.Jpeg); + diff2Size += ms.ToArray().Length; + } + } + } + Debug.WriteLine($"Diff2 size: {diff2Size}"); + + + + sw.Restart(); + var diffs3 = ImageUtils.GetDiffAreas3(frame1, frame2, false); + Debug.WriteLine($"Diff3 time: {sw.Elapsed.TotalMilliseconds}"); + diffSize = 0; + foreach (var x in diffs3) + { + using (var tempImage = (Bitmap)frame1.Clone(new Rectangle(x.X, x.Y, x.Width, x.Height), PixelFormat.Format32bppArgb)) + { + using (var ms = new MemoryStream()) + { + tempImage.Save(ms, ImageFormat.Jpeg); + diffSize += ms.ToArray().Length; + } + } + } + Debug.WriteLine($"Diff3 size: {diffSize}"); + + + sw.Restart(); + var diffImage = ImageUtils.GetImageDiff(frame1, frame2, false, out var hadChanges); + Debug.WriteLine($"Diff Image time: {sw.Elapsed.TotalMilliseconds}"); + sw.Restart(); + using (var ms = new MemoryStream()) + { + diffImage.Save(ms, ImageFormat.Png); + Debug.WriteLine($"Diff image size: {ms.ToArray().Length}"); + } + Debug.WriteLine($"Diff Image encode time: {sw.Elapsed.TotalMilliseconds}"); + + + + sw.Restart(); + var gifImage = (Bitmap)diffImage.Clone(); + using (var ms = new MemoryStream()) + { + gifImage.MakeTransparent(Color.FromArgb(0, 0, 0, 0)); + gifImage.Save(ms, ImageFormat.Gif); + gifImage.Save(@"C:\Users\trans\Desktop\Test.gif"); + Debug.WriteLine($"GIF image size: {ms.ToArray().Length}"); + } + Debug.WriteLine($"GIF Image encode time: {sw.Elapsed.TotalMilliseconds}"); + + //sw.Restart(); + //using (var ms = new MemoryStream()) + //{ + // diffImage.Save(ms, ImageFormat.Jpeg); + // ms.Seek(0, SeekOrigin.Begin); + // var pngEncoder = new PngEncoder() { CompressionLevel = PngCompressionLevel.BestSpeed }; + // using (var ms2 = new MemoryStream()) + // { + // SixLabors.ImageSharp.Image.Load(ms).Save(ms2, pngEncoder); + // Debug.WriteLine($"ImageSharp size: {ms2.ToArray().Length}"); + // } + //} + //Debug.WriteLine($"ImageSharp encode time: {sw.Elapsed.TotalMilliseconds}"); + + + //sw.Restart(); + //using (var ms = new MemoryStream()) + //{ + // diffImage.Save(ms, ImageFormat.Jpeg); + // ms.Seek(0, SeekOrigin.Begin); + // using (var ms2 = new MemoryStream()) + // { + // Aspose.Imaging.Image.Load(ms).Save(ms2, new PngOptions() { CompressionLevel = 5 }); + // Debug.WriteLine($"Aspose size: {ms2.ToArray().Length}"); + // } + //} + //Debug.WriteLine($"Aspose encode time: {sw.Elapsed.TotalMilliseconds}"); + + + + + //sw.Restart(); + //var drawingBytes = ImageUtils.EncodeBitmap(frame1, null); + //Debug.WriteLine($"Drawing Encoder time: {sw.Elapsed.TotalMilliseconds}"); + //Debug.WriteLine($"Drawing encoder size: {drawingBytes.Length}"); + + //sw.Restart(); + //using (var ms = new MemoryStream()) + //{ + // frame1.Clone(new Rectangle(0, 0, 500, 500), PixelFormat.Format32bppArgb) + // .Save(ms, GetEncoder(ImageFormat.Jpeg), encoderParams); + // Debug.WriteLine($"Jpeg encode time: {sw.Elapsed.TotalMilliseconds}"); + // Debug.WriteLine($"Jpeg encode size: {ms.ToArray().Length}"); + //} + + //var factory = new ImageProcessor.ImageFactory(); + //sw.Restart(); + //using (var ms = new MemoryStream()) + //{ + // var webPFormat = new ImageProcessor.Plugins.WebP.Imaging.Formats.WebPFormat(); + // factory.Load(diffImage) + // .Format(webPFormat) + // .Quality(60) + // .Save(ms); + + // Debug.WriteLine($"Webp encode time: {sw.Elapsed.TotalMilliseconds}"); + // Debug.WriteLine($"Webp encode size: {ms.ToArray().Length}"); + //} + + Debug.WriteLine($"\n"); + } + } + [TestInitialize] public void Init() { @@ -114,5 +282,26 @@ namespace Remotely.Tests ServiceContainer.Instance = serviceCollection.BuildServiceProvider(); } + + private Bitmap GetFrame(string frameFileName) + { + using (var mrs = Assembly.GetExecutingAssembly().GetManifestResourceStream($"Remotely.Tests.Resources.{frameFileName}.jpg")) + { + var resourceImage = (Bitmap)Bitmap.FromStream(mrs); + + if (resourceImage.PixelFormat != PixelFormat.Format32bppArgb) + { + return resourceImage.Clone(new Rectangle(0, 0, resourceImage.Width, resourceImage.Height), PixelFormat.Format32bppArgb); + } + return resourceImage; + } + } + + private ImageCodecInfo GetEncoder(ImageFormat format) + { + var codecs = ImageCodecInfo.GetImageEncoders(); + + return codecs.FirstOrDefault(x => x.FormatID == format.Guid); + } } } diff --git a/Tests/Resources/Frame1.jpg b/Tests/Resources/Frame1.jpg new file mode 100644 index 00000000..4b0bace0 Binary files /dev/null and b/Tests/Resources/Frame1.jpg differ diff --git a/Tests/Resources/Frame2.jpg b/Tests/Resources/Frame2.jpg new file mode 100644 index 00000000..8f69e739 Binary files /dev/null and b/Tests/Resources/Frame2.jpg differ diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 31b3a54b..fba4675a 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -12,6 +12,16 @@ AnyCPU;x64;x86 + + + + + + + + + +