mirrors/Remotely
3
0
Fork 0
mirror of https://github.com/immense/Remotely.git synced 2025-10-26 11:27:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merged PR 3: Frame diffing and canvas drawing experiments

Browse Source
This commit is contained in:
Jared Goodwin 2020-12-24 19:42:52 +00:00 committed by Jared Goodwin
parent 4c43817aea
commit 03a93f2574
20 changed files with 754 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/deploy-to-iis.yml vendored
View File

@ -145,11 +145,6 @@ jobs:
Write-Host "Setting current version to $CurrentVersion."
# Run the Publish script to build clients and server.
- name: Run Publish script
shell: powershell
run: .\Utilities\Publish.ps1 -CertificatePath "$env:GITHUB_WORKSPACE\GitHubActionsWorkflow.pfx" -CertificatePassword $env:PfxKey -Hostname $env:SiteUrl -CurrentVersion $env:CurrentVersion
# Create MSDeploy Publishing Profile
- name: Create MSDeploy Profile
shell: powershell
@ -172,7 +167,7 @@ jobs:
<MSDeployPublishMethod>WMSVC</MSDeployPublishMethod>
<EnableMSDeployBackup>True</EnableMSDeployBackup>
<UserName>$env:MsDeployUsername</UserName>
<TargetFramework>netcoreapp3.1</TargetFramework>
<TargetFramework>net5.0</TargetFramework>
<SelfContained>true</SelfContained>
<AllowUntrustedCertificate>true</AllowUntrustedCertificate>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
@ -182,8 +177,19 @@ jobs:
New-Item -Path "$env:GITHUB_WORKSPACE\Server\Properties\PublishProfiles\" -ItemType Directory -Force
Set-Content -Path "$env:GITHUB_WORKSPACE\Server\Properties\PublishProfiles\DeployIIS.pubxml" -Value $PublishProfile -Force
# Run the Publish script to build clients and server.
- name: Run Publish script
shell: powershell
run: |
.\Utilities\Publish.ps1 -CertificatePath "$env:GITHUB_WORKSPACE\GitHubActionsWorkflow.pfx" -CertificatePassword $env:PfxKey -Hostname $env:SiteUrl -CurrentVersion $env:CurrentVersion -RID win-x64 -OutDir "$env:GITHUB_WORKSPACE\publish"
# Upload build artifact to be deployed from Ubuntu runner
- name: Upload build artifact
uses: actions/upload-artifact@v2
with:
path: ./publish/
# Uncomment the below to enable deployment to the server.
# Publish server to IIS
- name: Publish
run: |

17
Desktop.Core/Models/CaptureFrame.cs Normal file
View File

@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Remotely.Desktop.Core.Models
{
public class CaptureFrame
{
public byte[] EncodedImageBytes { get; init; }
public int Top { get; init; }
public int Left { get; init; }
public int Height { get; init; }
public int Width { get; init; }
}
}

88
Desktop.Core/Services/ScreenCaster.cs
View File

@ -1,10 +1,13 @@
using Microsoft.Extensions.DependencyInjection;
using Remotely.Desktop.Core.Enums;
using Remotely.Desktop.Core.Interfaces;
using Remotely.Desktop.Core.Models;
using Remotely.Desktop.Core.Utilities;
using Remotely.Shared.Helpers;
using Remotely.Shared.Models;
using Remotely.Shared.Utilities;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
@ -87,20 +90,29 @@ namespace Remotely.Desktop.Core.Services
{
if (initialFrame != null)
{
await viewer.SendScreenCapture(
ImageUtils.EncodeBitmap(initialFrame, viewer.EncoderParams),
viewer.Capturer.CurrentScreenBounds.Left,
viewer.Capturer.CurrentScreenBounds.Top,
viewer.Capturer.CurrentScreenBounds.Width,
viewer.Capturer.CurrentScreenBounds.Height);
await viewer.SendScreenCapture(new CaptureFrame[]
{
new CaptureFrame()
{
EncodedImageBytes = ImageUtils.EncodeBitmap(initialFrame, viewer.EncoderParams),
Left = viewer.Capturer.CurrentScreenBounds.Left,
Top = viewer.Capturer.CurrentScreenBounds.Top,
Width = viewer.Capturer.CurrentScreenBounds.Width,
Height = viewer.Capturer.CurrentScreenBounds.Height
}
});
}
}
if (EnvironmentHelper.IsWindows)
{
await viewer.InitializeWebRtc();
}
// Wait until the first image is received.
TaskHelper.DelayUntil(() => !viewer.PendingSentFrames.Any(), TimeSpan.MaxValue);
while (!viewer.DisconnectRequested && viewer.IsConnected)
{
try
@ -137,15 +149,20 @@ namespace Remotely.Desktop.Core.Services
currentFrame?.Dispose();
currentFrame = viewer.Capturer.GetNextFrame();
var diffAreas = ImageUtils.GetDiffAreas(currentFrame, previousFrame, viewer.Capturer.CaptureFullscreen);
var diffAreas = ImageUtils.GetDiffAreas2(currentFrame, previousFrame, viewer.Capturer.CaptureFullscreen);
if (diffAreas.Count == 0)
if (!diffAreas.Any())
{
continue;
}
viewer.Capturer.CaptureFullscreen = false;
var frameClone = (Bitmap)currentFrame.Clone();
Debug.WriteLine($"Sending {diffAreas.Count} frames.");
await sendFramesLock.WaitAsync();
SendFrames((Bitmap)currentFrame.Clone(), diffAreas, viewer, sendFramesLock);
SendFrames(frameClone, diffAreas, viewer, sendFramesLock);
}
catch (Exception ex)
{
@ -172,28 +189,65 @@ namespace Remotely.Desktop.Core.Services
}
}
private void SendFrames(Bitmap currentFrame, List<Rectangle> diffAreas, Viewer viewer, SemaphoreSlim sendFramesLock)
private void SendFrame(Bitmap diffImage, Viewer viewer, SemaphoreSlim sendFramesLock)
{
_ = Task.Run(async () =>
{
try
{
var encodedImageBytes = ImageUtils.EncodeGif(diffImage);
if (encodedImageBytes?.Length > 0)
{
var frames = new List<CaptureFrame>()
{
new CaptureFrame()
{
EncodedImageBytes = encodedImageBytes,
Top = 0,
Left = 0,
Width = diffImage.Width,
Height = diffImage.Height,
}
};
await viewer.SendScreenCapture(frames);
}
}
finally
{
sendFramesLock.Release();
diffImage.Dispose();
}
});
}
private static void SendFrames(Bitmap currentFrame, ICollection<Rectangle> diffAreas, Viewer viewer, SemaphoreSlim sendFramesLock)
{
_ = Task.Run(async () =>
{
try
{
var frames = new List<CaptureFrame>();
foreach (var diffArea in diffAreas)
{
using var newImage = currentFrame.Clone(diffArea, PixelFormat.Format32bppArgb);
if (viewer.Capturer.CaptureFullscreen)
{
viewer.Capturer.CaptureFullscreen = false;
}
var encodedImageBytes = ImageUtils.EncodeBitmap(newImage, viewer.EncoderParams);
if (encodedImageBytes?.Length > 0)
{
await viewer.SendScreenCapture(encodedImageBytes, diffArea.Left, diffArea.Top, diffArea.Width, diffArea.Height);
frames.Add(new CaptureFrame()
{
EncodedImageBytes = encodedImageBytes,
Top = diffArea.Top,
Left = diffArea.Left,
Width = diffArea.Width,
Height = diffArea.Height,
});
}
}
};
await viewer.SendScreenCapture(frames);
}
finally
{

63
Desktop.Core/Services/Viewer.cs
View File

@ -1,4 +1,5 @@
using Remotely.Desktop.Core.Interfaces;
using Remotely.Desktop.Core.Models;
using Remotely.Desktop.Core.ViewModels;
using Remotely.Shared.Helpers;
using Remotely.Shared.Models;
@ -6,7 +7,9 @@ using Remotely.Shared.Models.RemoteControlDtos;
using Remotely.Shared.Utilities;
using Remotely.Shared.Win32;
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
@ -17,7 +20,7 @@ namespace Remotely.Desktop.Core.Services
public class Viewer : IDisposable
{
private readonly int _defaultImageQuality = 60;
private int _imageQuality;
private long _imageQuality;
private DateTimeOffset _lastQualityAdjustment;
public Viewer(ICasterSocket casterSocket,
IScreenCapturer screenCapturer,
@ -43,7 +46,7 @@ namespace Remotely.Desktop.Core.Services
public bool DisconnectRequested { get; set; }
public EncoderParameters EncoderParams { get; private set; }
public bool HasControl { get; set; } = true;
public int ImageQuality
public long ImageQuality
{
get
{
@ -247,39 +250,63 @@ namespace Remotely.Desktop.Core.Services
() => CasterSocket.SendDtoToViewer(dto, ViewerConnectionID));
}
public async Task SendScreenCapture(byte[] encodedImageBytes, int left, int top, int width, int height)
public async Task SendScreenCapture(IEnumerable<CaptureFrame> screenFrame)
{
PendingSentFrames.Enqueue(DateTimeOffset.Now);
for (var i = 0; i < encodedImageBytes.Length; i += 50_000)
foreach (var frame in screenFrame)
{
var dto = new CaptureFrameDto()
var left = frame.Left;
var top = frame.Top;
var width = frame.Width;
var height = frame.Height;
for (var i = 0; i < frame.EncodedImageBytes.Length; i += 50_000)
{
var dto = new CaptureFrameDto()
{
Left = left,
Top = top,
Width = width,
Height = height,
EndOfFrame = false,
ImageBytes = frame.EncodedImageBytes.Skip(i).Take(50_000).ToArray(),
ImageQuality = _imageQuality,
EndOfCapture = false
};
await SendToViewer(() => RtcSession.SendDto(dto),
() => CasterSocket.SendDtoToViewer(dto, ViewerConnectionID));
}
var endOfFrameDto = new CaptureFrameDto()
{
Left = left,
Top = top,
Width = width,
Height = height,
EndOfFrame = false,
ImageBytes = encodedImageBytes.Skip(i).Take(50_000).ToArray(),
ImageQuality = _imageQuality
EndOfFrame = true,
ImageQuality = _imageQuality,
EndOfCapture = false
};
await SendToViewer(() => RtcSession.SendDto(dto),
() => CasterSocket.SendDtoToViewer(dto, ViewerConnectionID));
await SendToViewer(() => RtcSession.SendDto(endOfFrameDto),
() => CasterSocket.SendDtoToViewer(endOfFrameDto, ViewerConnectionID));
}
var endOfFrameDto = new CaptureFrameDto()
var endofCaptureDto = new CaptureFrameDto()
{
Left = left,
Top = top,
Width = width,
Height = height,
Left = 0,
Top = 0,
Width = 0,
Height = 0,
EndOfFrame = true,
ImageQuality = _imageQuality
ImageQuality = _imageQuality,
EndOfCapture = true
};
await SendToViewer(() => RtcSession.SendDto(endOfFrameDto),
() => CasterSocket.SendDtoToViewer(endOfFrameDto, ViewerConnectionID));
await SendToViewer(() => RtcSession.SendDto(endofCaptureDto),
() => CasterSocket.SendDtoToViewer(endofCaptureDto, ViewerConnectionID));
}
public async Task SendScreenData(string selectedScreen, string[] displayNames)

295
Desktop.Core/Utilities/ImageUtils.cs
View File

@ -1,23 +1,36 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
namespace Remotely.Desktop.Core.Utilities
{
public class ImageUtils
{
public static ImageCodecInfo JpegEncoder { get; } = ImageCodecInfo.GetImageEncoders().FirstOrDefault(x => x.FormatID == ImageFormat.Jpeg.Guid);
public static ImageCodecInfo GifEncoder { get; } = ImageCodecInfo.GetImageEncoders().FirstOrDefault(x => x.FormatID == ImageFormat.Gif.Guid);
public static byte[] EncodeBitmap(Bitmap bitmap, EncoderParameters encoderParams)
{
using var ms = new MemoryStream();
bitmap.Save(ms, JpegEncoder, encoderParams);
return ms.ToArray();
}
public static byte[] EncodeGif(Bitmap diffImage)
{
diffImage.MakeTransparent(Color.FromArgb(0, 0, 0, 0));
using var ms = new MemoryStream();
diffImage.Save(ms, ImageFormat.Gif);
return ms.ToArray();
}
public static List<Rectangle> GetDiffAreas(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen)
{
var changes = new List<Rectangle>();
@ -106,7 +119,7 @@ namespace Remotely.Desktop.Core.Utilities
}
if (!changeOnCurrentRow &&
changeOnPreviousRow &&
left <= right &&
left <= right &&
top <= bottom)
{
AddChangeToList(changes, left, top, right, bottom, width, height);
@ -138,10 +151,267 @@ namespace Remotely.Desktop.Core.Utilities
}
}
public static Bitmap GetImageDiff(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen)
public static ICollection<Rectangle> GetDiffAreas2(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen)
{
if (currentFrame == null || previousFrame == null)
{
return Array.Empty<Rectangle>();
}
if (captureFullscreen)
{
return new Rectangle[] { new Rectangle(new Point(0, 0), currentFrame.Size) };
}
if (currentFrame.Height != previousFrame.Height || currentFrame.Width != previousFrame.Width)
{
throw new Exception("Bitmaps are not of equal dimensions.");
}
if (currentFrame.PixelFormat != previousFrame.PixelFormat)
{
throw new Exception("Bitmaps are not the same format.");
}
var width = currentFrame.Width;
var height = currentFrame.Height;
BitmapData bd1 = null;
BitmapData bd2 = null;
var bytesPerPixel = Bitmap.GetPixelFormatSize(currentFrame.PixelFormat) / 8;
var numberOfPixels = width * height;
var totalSize = numberOfPixels * bytesPerPixel;
var changes = new ConcurrentQueue<Rectangle>();
try
{
bd1 = previousFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, currentFrame.PixelFormat);
bd2 = currentFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, previousFrame.PixelFormat);
unsafe
{
byte* scan1 = (byte*)bd1.Scan0.ToPointer();
byte* scan2 = (byte*)bd2.Scan0.ToPointer();
var gridColumnWidth = width % 8 == 0 ? width / 8 :
width % 2 == 0 ? width / 2 :
width;
var gridRowHeight = height % 9 == 0 ? height / 9 :
height % 10 == 0 ? height / 10 :
height % 4 == 0 ? height / 4 :
height;
var gridColumns = Enumerable.Range(0, width).Where(i => i % gridColumnWidth == 0);
var gridRows = Enumerable.Range(0, height).Where(i => i % gridRowHeight == 0);
Parallel.ForEach(gridColumns, gridColumn =>
{
Parallel.ForEach(gridRows, gridRow =>
{
int left = int.MaxValue;
int top = int.MaxValue;
int right = int.MinValue;
int bottom = int.MinValue;
for (var row = 0; row < gridRowHeight; row++)
{
for (var col = 0; col < gridColumnWidth; col++)
{
var pixelLeft = gridColumn + col;
var pixelTop = gridRow + row;
var rowIndex = pixelTop * width * bytesPerPixel;
var columnIndex = pixelLeft * bytesPerPixel;
var i = rowIndex + columnIndex;
byte* data1 = scan1 + i;
byte* data2 = scan2 + i;
if (data1[0] != data2[0] ||
data1[1] != data2[1] ||
data1[2] != data2[2] ||
data1[3] != data2[3])
{
if (pixelTop < top)
{
top = pixelTop;
}
if (pixelTop > bottom)
{
bottom = pixelTop;
}
if (pixelLeft < left)
{
left = pixelLeft;
}
if (pixelLeft > right)
{
right = pixelLeft;
}
}
}
}
if (left <= right && top <= bottom)
{
AddChangeToList(changes, left, top, right, bottom, width, height);
}
});
});
return changes.ToArray();
}
}
catch
{
return changes.ToArray();
}
finally
{
currentFrame.UnlockBits(bd1);
previousFrame.UnlockBits(bd2);
}
}
public static ICollection<Rectangle> GetDiffAreas3(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen)
{
if (currentFrame == null || previousFrame == null)
{
return Array.Empty<Rectangle>();
}
if (captureFullscreen)
{
return new Rectangle[] { new Rectangle(new Point(0, 0), currentFrame.Size) };
}
if (currentFrame.Height != previousFrame.Height || currentFrame.Width != previousFrame.Width)
{
throw new Exception("Bitmaps are not of equal dimensions.");
}
if (currentFrame.PixelFormat != previousFrame.PixelFormat)
{
throw new Exception("Bitmaps are not the same format.");
}
var width = currentFrame.Width;
var height = currentFrame.Height;
BitmapData bd1 = null;
BitmapData bd2 = null;
var bytesPerPixel = Bitmap.GetPixelFormatSize(currentFrame.PixelFormat) / 8;
var numberOfPixels = width * height;
var totalSize = numberOfPixels * bytesPerPixel;
var changes = new ConcurrentQueue<Rectangle>();
try
{
bd1 = previousFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, currentFrame.PixelFormat);
bd2 = currentFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, previousFrame.PixelFormat);
unsafe
{
byte* scan1 = (byte*)bd1.Scan0.ToPointer();
byte* scan2 = (byte*)bd2.Scan0.ToPointer();
var gridRowHeight = height % 9 == 0 ? height / 9 :
height % 10 == 0 ? height / 10 :
height % 4 == 0 ? height / 4 :
height;
var gridRows = Enumerable.Range(0, height).Where(i => i % gridRowHeight == 0);
Parallel.ForEach(gridRows, gridRow =>
{
int left = int.MaxValue;
int top = int.MaxValue;
int right = int.MinValue;
int bottom = int.MinValue;
for (var row = 0; row < gridRowHeight; row++)
{
for (var col = 0; col < width; col++)
{
var pixelLeft = col;
var pixelTop = gridRow + row;
var rowIndex = pixelTop * width * bytesPerPixel;
var columnIndex = pixelLeft * bytesPerPixel;
var i = rowIndex + columnIndex;
byte* data1 = scan1 + i;
byte* data2 = scan2 + i;
if (data1[0] != data2[0] ||
data1[1] != data2[1] ||
data1[2] != data2[2] ||
data1[3] != data2[3])
{
if (pixelTop < top)
{
top = pixelTop;
}
if (pixelTop > bottom)
{
bottom = pixelTop;
}
if (pixelLeft < left)
{
left = pixelLeft;
}
if (pixelLeft > right)
{
right = pixelLeft;
}
}
}
}
if (left <= right && top <= bottom)
{
AddChangeToList(changes, left, top, right, bottom, width, height);
}
});
return changes.ToArray();
}
}
catch
{
return changes.ToArray();
}
finally
{
currentFrame.UnlockBits(bd1);
previousFrame.UnlockBits(bd2);
}
}
public static Bitmap GetImageDiff(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen, out bool hadChanges)
{
hadChanges = false;
if (currentFrame is null || previousFrame is null)
{
hadChanges = false;
return null;
}
if (captureFullscreen)
{
hadChanges = true;
return (Bitmap)currentFrame.Clone();
}
@ -185,6 +455,7 @@ namespace Remotely.Desktop.Core.Utilities
data1[2] != data2[2] ||
data1[3] != data2[3])
{
hadChanges = true;
data3[0] = data2[0];
data3[1] = data2[1];
data3[2] = data2[2];
@ -213,12 +484,24 @@ namespace Remotely.Desktop.Core.Utilities
{
// Bounding box is valid. Padding is necessary to prevent artifacts from
// moving windows.
left = Math.Max(left - 5, 0);
top = Math.Max(top - 5, 0);
right = Math.Min(right + 5, width);
bottom = Math.Min(bottom + 5, height);
left = Math.Max(left - 1, 0);
top = Math.Max(top - 1, 0);
right = Math.Min(right + 1, width);
bottom = Math.Min(bottom + 1, height);
changes.Add(new Rectangle(left, top, right - left, bottom - top));
}
private static void AddChangeToList(ConcurrentQueue<Rectangle> changes, int left, int top, int right, int bottom, int width, int height)
{
// Bounding box is valid. Padding is necessary to prevent artifacts from
// moving windows.
left = Math.Max(left - 1, 0);
top = Math.Max(top - 1, 0);
right = Math.Min(right + 1, width);
bottom = Math.Min(bottom + 1, height);
changes.Enqueue(new Rectangle(left, top, right - left, bottom - top));
}
}
}

29
Desktop.Win/Program.cs
View File

@ -35,7 +35,7 @@ namespace Remotely.Desktop.Win
}
}
}
[STAThread]
public static void Main(string[] args)
{
try
@ -89,19 +89,6 @@ namespace Remotely.Desktop.Win
}
}
private static void WaitForAppExit()
{
var appExitEvent = new ManualResetEventSlim();
App.Current.Dispatcher.Invoke(() =>
{
App.Current.Exit += (s, a) =>
{
appExitEvent.Set();
};
});
appExitEvent.Wait();
}
private static void BuildServices()
{
var serviceCollection = new ServiceCollection();
@ -165,6 +152,7 @@ namespace Remotely.Desktop.Win
private static async Task StartScreenCasting()
{
CursorIconWatcher = Services.GetRequiredService<ICursorIconWatcher>();
await CasterSocket.Connect(Conductor.Host);
@ -226,5 +214,18 @@ namespace Remotely.Desktop.Win
}
Logger.Write("Background UI apps started.");
}
private static void WaitForAppExit()
{
var appExitEvent = new ManualResetEventSlim();
App.Current.Dispatcher.Invoke(() =>
{
App.Current.Exit += (s, a) =>
{
appExitEvent.Set();
};
});
appExitEvent.Wait();
}
}
}

19
Desktop.Win/ViewModels/MainWindowViewModel.cs
View File

@ -25,8 +25,6 @@ namespace Remotely.Desktop.Win.ViewModels
private string _host;
private string _sessionID;
public static MainWindowViewModel Current { get; private set; }
public MainWindowViewModel()
{
Current = this;
@ -50,6 +48,7 @@ namespace Remotely.Desktop.Win.ViewModels
Conductor.ScreenCastRequested += ScreenCastRequested;
}
public static MainWindowViewModel Current { get; private set; }
public static IServiceProvider Services => ServiceContainer.Instance;
public ICommand ChangeServerCommand
@ -64,10 +63,6 @@ namespace Remotely.Desktop.Win.ViewModels
}
}
private Conductor Conductor { get; set; }
private ICasterSocket CasterSocket { get; set; }
private ICursorIconWatcher CursorIconWatcher { get; set; }
public ICommand ElevateToAdminCommand
{
get
@ -178,6 +173,12 @@ namespace Remotely.Desktop.Win.ViewModels
public ObservableCollection<Viewer> Viewers { get; } = new ObservableCollection<Viewer>();
private ICasterSocket CasterSocket { get; set; }
private Conductor Conductor { get; set; }
private ICursorIconWatcher CursorIconWatcher { get; set; }
public void CopyLink()
{
Clipboard.SetText($"{Host}/RemoteControl?sessionID={SessionID?.Replace(" ", "")}");
@ -192,7 +193,7 @@ namespace Remotely.Desktop.Win.ViewModels
public async Task Init()
{
SessionID = "Retrieving...";
Host = Config.GetConfig().Host;
while (string.IsNullOrWhiteSpace(Host))
@ -263,6 +264,10 @@ namespace Remotely.Desktop.Win.ViewModels
}
}
public void ShutdownApp()
{
Services.GetRequiredService<IShutdownService>().Shutdown();
}
private void Application_Exit(object sender, ExitEventArgs e)
{
App.Current.Dispatcher.Invoke(() =>

1
Desktop.Win/Views/MainWindow.xaml
View File

@ -10,6 +10,7 @@
WindowStyle="None"
ResizeMode="NoResize"
MouseLeftButtonDown="Window_MouseLeftButtonDown"
Closing="Window_Closing"
Loaded="Window_Loaded" Icon="/Assets/Remotely_Icon.png">
<Window.Resources>
<DrawingBrush x:Key="GearBrush">

7
Desktop.Win/Views/MainWindow.xaml.cs
View File

@ -47,7 +47,7 @@ namespace Remotely.Desktop.Win.Views
private void MinimizeButton_Click(object sender, RoutedEventArgs e)
{
this.WindowState = WindowState.Minimized;
WindowState = WindowState.Minimized;
}
private void OptionsButton_Click(object sender, RoutedEventArgs e)
@ -55,6 +55,11 @@ namespace Remotely.Desktop.Win.Views
(sender as Button).ContextMenu.IsOpen = true;
}
private void Window_Closing(object sender, CancelEventArgs e)
{
ViewModel?.ShutdownApp();
}
private async void Window_Loaded(object sender, RoutedEventArgs e)
{
if (!DesignerProperties.GetIsInDesignMode(this) &&

78
Server/wwwroot/src/RemoteControl/DtoMessageHandler.ts
View File

@ -17,18 +17,17 @@ import {
} from "./Interfaces/Dtos.js";
import { ReceiveFile } from "./FileTransferService.js";
export class DtoMessageHandler {
MessagePack: any = window['MessagePack'];
PartialCaptureFrames: Uint8Array[] = [];
ParseBinaryMessage(data: ArrayBuffer) {
PartialCaptures: Record<string, CaptureFrameDto[]> = {};
async ParseBinaryMessage(data: ArrayBuffer) {
var model = this.MessagePack.decode(data) as BaseDto;
switch (model.DtoType) {
case BaseDtoType.AudioSample:
this.HandleAudioSample(model as unknown as AudioSampleDto);
break;
case BaseDtoType.CaptureFrame:
this.HandleCaptureFrame(model as unknown as CaptureFrameDto);
await this.HandleCaptureFrame(model as unknown as CaptureFrameDto);
break;
case BaseDtoType.ClipboardText:
this.HandleClipboardText(model as unknown as ClipboardTextDto);
@ -58,31 +57,72 @@ export class DtoMessageHandler {
HandleAudioSample(audioSample: AudioSampleDto) {
Sound.Play(audioSample.Buffer);
}
HandleCaptureFrame(captureFrame: CaptureFrameDto) {
async HandleCaptureFrame(captureFrame: CaptureFrameDto) {
if (UI.AutoQualityAdjustCheckBox.checked &&
Number(UI.QualitySlider.value) != captureFrame.ImageQuality) {
UI.QualitySlider.value = String(captureFrame.ImageQuality);
}
if (captureFrame.EndOfFrame) {
if (captureFrame.EndOfCapture) {
ViewerApp.MessageSender.SendFrameReceived();
var url = window.URL.createObjectURL(new Blob(this.PartialCaptureFrames));
var img = document.createElement("img");
img.onload = () => {
UI.Screen2DContext.drawImage(img,
captureFrame.Left,
captureFrame.Top,
captureFrame.Width,
captureFrame.Height);
window.URL.revokeObjectURL(url);
};
img.src = url;
this.PartialCaptureFrames = [];
Object.keys(this.PartialCaptures).forEach(async x => {
let partial = this.PartialCaptures[x];
let firstFrame = partial[0];
let frameBytes = partial.map(x => x.ImageBytes);
let bitmap = await createImageBitmap(new Blob(frameBytes));
UI.Screen2DContext.drawImage(bitmap,
firstFrame.Left,
firstFrame.Top,
firstFrame.Width,
firstFrame.Height);
bitmap.close();
})
this.PartialCaptures = {};
}
//else if (captureFrame.EndOfFrame) {
// let key = `${captureFrame.Left},${captureFrame.Top}`;
// let frameBytes = this.PartialCaptures[key].map(x => x.ImageBytes);
// //var url = window.URL.createObjectURL(new Blob(frameBytes));
// //var img = document.createElement("img");
// //img.onload = () => {
// // UI.StagingRenderer.drawImage(img,
// // captureFrame.Left,
// // captureFrame.Top,
// // captureFrame.Width,
// // captureFrame.Height);
// // window.URL.revokeObjectURL(url);
// //};
// //img.src = url;
// let bitmap = await createImageBitmap(new Blob(frameBytes));
// UI.StagingRenderer.drawImage(bitmap,
// captureFrame.Left,
// captureFrame.Top,
// captureFrame.Width,
// captureFrame.Height);
// bitmap.close();
//}
else {
this.PartialCaptureFrames.push(captureFrame.ImageBytes);
let key = `${captureFrame.Left},${captureFrame.Top}`;
if (this.PartialCaptures[key]) {
this.PartialCaptures[key].push(captureFrame);
}
else {
this.PartialCaptures[key] = [captureFrame];
}
}
}
HandleClipboardText(clipboardText: ClipboardTextDto) {
ViewerApp.ClipboardWatcher.SetClipboardText(clipboardText.ClipboardText);
ShowMessage("Clipboard updated.");

4
Server/wwwroot/src/RemoteControl/InputEventHandlers.ts
View File

@ -567,10 +567,12 @@ export function ApplyInputHandlers() {
if (document.querySelector("input:focus") || document.querySelector("textarea:focus")) {
return;
}
e.preventDefault();
if (ViewerApp.ViewOnlyMode) {
return;
}
if (!e.ctrlKey || !e.shiftKey || e.key.toLowerCase() != "i") {
e.preventDefault();
}
ViewerApp.MessageSender.SendKeyDown(e.key);
});
window.addEventListener("keyup", function (e) {

1
Server/wwwroot/src/RemoteControl/Interfaces/Dtos.ts
View File

@ -18,6 +18,7 @@ export interface AudioSampleDto extends BaseDto {
export interface CaptureFrameDto extends BaseDto {
EndOfFrame: boolean;
EndOfCapture: boolean;
Left: number;
Top: number;
Width: number;

2
Server/wwwroot/src/RemoteControl/RtcSession.ts
View File

@ -46,7 +46,7 @@ export class RtcSession {
};
this.DataChannel.onmessage = async (ev) => {
var data = ev.data as ArrayBuffer;
ViewerApp.DtoMessageHandler.ParseBinaryMessage(data);
await ViewerApp.DtoMessageHandler.ParseBinaryMessage(data);
};
this.DataChannel.onopen = (ev) => {

10
Server/wwwroot/src/RemoteControl/UI.ts
View File

@ -3,6 +3,9 @@ import { ConvertUInt8ArrayToBase64 } from "../Shared/Utilities.js";
import { WindowsSession } from "../Shared/Models/WindowsSession.js";
import { WindowsSessionType } from "../Shared/Enums/WindowsSessionType.js";
const offscreenCanvas = document.createElement("canvas");
var renderId: number;
export var AudioButton = document.getElementById("audioButton") as HTMLButtonElement;
export var MenuButton = document.getElementById("menuButton") as HTMLButtonElement;
export var MenuFrame = document.getElementById("menuFrame") as HTMLDivElement;
@ -49,6 +52,8 @@ export var RecordSessionButton = document.getElementById("recordSessionButton")
export var DownloadRecordingButton = document.getElementById("downloadRecordingButton") as HTMLButtonElement;
export var ViewOnlyButton = document.getElementById("viewOnlyButton") as HTMLButtonElement;
export var FullScreenButton = document.getElementById("fullScreenButton") as HTMLButtonElement;
export var StagingCanvas = offscreenCanvas as HTMLCanvasElement;
export var StagingRenderer = offscreenCanvas.getContext("2d");
export function GetCurrentViewer(): HTMLElement {
if (ScreenViewer.hasAttribute("hidden")) {
@ -96,15 +101,20 @@ export function Prompt(promptMessage: string): Promise<string> {
});
}
export function SetScreenSize(width: number, height: number) {
ScreenViewer.width = width;
ScreenViewer.height = height;
StagingCanvas.width = width;
StagingCanvas.height = height;
Screen2DContext.clearRect(0, 0, width, height);
StagingRenderer.clearRect(0, 0, width, height);
}
export function ToggleConnectUI(shown: boolean) {
if (shown) {
Screen2DContext.clearRect(0, 0, ScreenViewer.width, ScreenViewer.height);
StagingRenderer.clearRect(0, 0, ScreenViewer.width, ScreenViewer.height);
ScreenViewer.setAttribute("hidden", "hidden");
VideoScreenViewer.setAttribute("hidden", "hidden");
ConnectBox.style.removeProperty("display");

30
Server/wwwroot/src/RemoteControl/ViewerHubConnection.ts
View File

@ -73,8 +73,8 @@ export class ViewerHubConnection {
private ApplyMessageHandlers(hubConnection) {
hubConnection.on("SendDtoToBrowser", (dto: ArrayBuffer) => {
ViewerApp.DtoMessageHandler.ParseBinaryMessage(dto);
hubConnection.on("SendDtoToBrowser", async (dto: ArrayBuffer) => {
await ViewerApp.DtoMessageHandler.ParseBinaryMessage(dto);
});
hubConnection.on("ClipboardTextChanged", (clipboardText: string) => {
ViewerApp.ClipboardWatcher.SetClipboardText(clipboardText);
@ -86,33 +86,7 @@ export class ViewerHubConnection {
hubConnection.on("ScreenSize", (width: number, height: number) => {
UI.SetScreenSize(width, height);
});
hubConnection.on("ScreenCapture", (buffer: Uint8Array,
left: number,
top: number,
width: number,
height: number,
imageQuality: number,
endOfFrame: boolean) => {
if (UI.AutoQualityAdjustCheckBox.checked && Number(UI.QualitySlider.value) != imageQuality) {
UI.QualitySlider.value = String(imageQuality);
}
if (endOfFrame) {
this.SendDtoToClient(new GenericDto(BaseDtoType.FrameReceived));
var url = window.URL.createObjectURL(new Blob(this.PartialCaptureFrames));
var img = document.createElement("img");
img.onload = () => {
UI.Screen2DContext.drawImage(img, left, top, width, height);
window.URL.revokeObjectURL(url);
};
img.src = url;
this.PartialCaptureFrames = [];
}
else {
this.PartialCaptureFrames.push(buffer);
}
});
hubConnection.on("ConnectionFailed", () => {
UI.ConnectButton.removeAttribute("disabled");
UI.StatusMessage.innerHTML = "Connection failed or was denied.";

3
Shared/Models/RemoteControlDtos/CaptureFrameDto.cs
View File

@ -9,6 +9,9 @@ namespace Remotely.Shared.Models.RemoteControlDtos
[DataMember(Name = "DtoType")]
public new BaseDtoType DtoType { get; } = BaseDtoType.CaptureFrame;
[DataMember(Name = "EndOfCapture")]
public bool EndOfCapture { get; set; }
[DataMember(Name = "EndOfFrame")]
public bool EndOfFrame { get; set; }

203
Tests/ManualTests.cs
View File

@ -5,17 +5,21 @@ using Moq;
using Remotely.Desktop.Core;
using Remotely.Desktop.Core.Interfaces;
using Remotely.Desktop.Core.Services;
using Remotely.Desktop.Core.Utilities;
using Remotely.Desktop.Win.Services;
using Remotely.Shared.Models;
using Remotely.Shared.Models.RemoteControlDtos;
using Remotely.Shared.Utilities;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Media.Imaging;
namespace Remotely.Tests
{
@ -52,12 +56,10 @@ namespace Remotely.Tests
ViewerID = "asdf"
};
var timeout = Debugger.IsAttached ?
20_000 :
5_000;
_ = Task.Run(async () => await _screenCaster.BeginScreenCasting(request));
var timeout = 5_000;
await Task.Delay(timeout);
_viewer.DisconnectRequested = true;
@ -68,6 +70,172 @@ namespace Remotely.Tests
_viewer.Dispose();
}
[TestMethod]
#if !DEBUG
[Ignore("Manual test.")]
#endif
public void EncodingTests()
{
for (var i = 0; i < 2; i++)
{
var encoderParams = new EncoderParameters()
{
Param = new EncoderParameter[]
{
new EncoderParameter(Encoder.Quality, 60L)
}
};
using var frame1 = GetFrame("Frame1");
using var frame2 = GetFrame("Frame2");
var sw = Stopwatch.StartNew();
sw.Restart();
var diffs = ImageUtils.GetDiffAreas(frame1, frame2, false);
Debug.WriteLine($"Diff time: {sw.Elapsed.TotalMilliseconds}");
var diffSize = 0;
foreach (var x in diffs)
{
using (var tempImage = (Bitmap)frame1.Clone(new Rectangle(x.X, x.Y, x.Width, x.Height), PixelFormat.Format32bppArgb))
{
using (var ms = new MemoryStream())
{
tempImage.Save(ms, ImageFormat.Jpeg);
diffSize += ms.ToArray().Length;
}
}
}
Debug.WriteLine($"Diff size: {diffSize}");
sw.Restart();
var diff2Size = 0;
var diffs2 = ImageUtils.GetDiffAreas2(frame1, frame2, false);
Debug.WriteLine($"Diff2 time: {sw.Elapsed.TotalMilliseconds}");
foreach (var x in diffs2)
{
using (var tempImage = (Bitmap)frame1.Clone(new Rectangle(x.X, x.Y, x.Width, x.Height), PixelFormat.Format32bppArgb))
{
using (var ms = new MemoryStream())
{
tempImage.Save(ms, ImageFormat.Jpeg);
diff2Size += ms.ToArray().Length;
}
}
}
Debug.WriteLine($"Diff2 size: {diff2Size}");
sw.Restart();
var diffs3 = ImageUtils.GetDiffAreas3(frame1, frame2, false);
Debug.WriteLine($"Diff3 time: {sw.Elapsed.TotalMilliseconds}");
diffSize = 0;
foreach (var x in diffs3)
{
using (var tempImage = (Bitmap)frame1.Clone(new Rectangle(x.X, x.Y, x.Width, x.Height), PixelFormat.Format32bppArgb))
{
using (var ms = new MemoryStream())
{
tempImage.Save(ms, ImageFormat.Jpeg);
diffSize += ms.ToArray().Length;
}
}
}
Debug.WriteLine($"Diff3 size: {diffSize}");
sw.Restart();
var diffImage = ImageUtils.GetImageDiff(frame1, frame2, false, out var hadChanges);
Debug.WriteLine($"Diff Image time: {sw.Elapsed.TotalMilliseconds}");
sw.Restart();
using (var ms = new MemoryStream())
{
diffImage.Save(ms, ImageFormat.Png);
Debug.WriteLine($"Diff image size: {ms.ToArray().Length}");
}
Debug.WriteLine($"Diff Image encode time: {sw.Elapsed.TotalMilliseconds}");
sw.Restart();
var gifImage = (Bitmap)diffImage.Clone();
using (var ms = new MemoryStream())
{
gifImage.MakeTransparent(Color.FromArgb(0, 0, 0, 0));
gifImage.Save(ms, ImageFormat.Gif);
gifImage.Save(@"C:\Users\trans\Desktop\Test.gif");
Debug.WriteLine($"GIF image size: {ms.ToArray().Length}");
}
Debug.WriteLine($"GIF Image encode time: {sw.Elapsed.TotalMilliseconds}");
//sw.Restart();
//using (var ms = new MemoryStream())
//{
// diffImage.Save(ms, ImageFormat.Jpeg);
// ms.Seek(0, SeekOrigin.Begin);
// var pngEncoder = new PngEncoder() { CompressionLevel = PngCompressionLevel.BestSpeed };
// using (var ms2 = new MemoryStream())
// {
// SixLabors.ImageSharp.Image.Load(ms).Save(ms2, pngEncoder);
// Debug.WriteLine($"ImageSharp size: {ms2.ToArray().Length}");
// }
//}
//Debug.WriteLine($"ImageSharp encode time: {sw.Elapsed.TotalMilliseconds}");
//sw.Restart();
//using (var ms = new MemoryStream())
//{
// diffImage.Save(ms, ImageFormat.Jpeg);
// ms.Seek(0, SeekOrigin.Begin);
// using (var ms2 = new MemoryStream())
// {
// Aspose.Imaging.Image.Load(ms).Save(ms2, new PngOptions() { CompressionLevel = 5 });
// Debug.WriteLine($"Aspose size: {ms2.ToArray().Length}");
// }
//}
//Debug.WriteLine($"Aspose encode time: {sw.Elapsed.TotalMilliseconds}");
//sw.Restart();
//var drawingBytes = ImageUtils.EncodeBitmap(frame1, null);
//Debug.WriteLine($"Drawing Encoder time: {sw.Elapsed.TotalMilliseconds}");
//Debug.WriteLine($"Drawing encoder size: {drawingBytes.Length}");
//sw.Restart();
//using (var ms = new MemoryStream())
//{
// frame1.Clone(new Rectangle(0, 0, 500, 500), PixelFormat.Format32bppArgb)
// .Save(ms, GetEncoder(ImageFormat.Jpeg), encoderParams);
// Debug.WriteLine($"Jpeg encode time: {sw.Elapsed.TotalMilliseconds}");
// Debug.WriteLine($"Jpeg encode size: {ms.ToArray().Length}");
//}
//var factory = new ImageProcessor.ImageFactory();
//sw.Restart();
//using (var ms = new MemoryStream())
//{
// var webPFormat = new ImageProcessor.Plugins.WebP.Imaging.Formats.WebPFormat();
// factory.Load(diffImage)
// .Format(webPFormat)
// .Quality(60)
// .Save(ms);
// Debug.WriteLine($"Webp encode time: {sw.Elapsed.TotalMilliseconds}");
// Debug.WriteLine($"Webp encode size: {ms.ToArray().Length}");
//}
Debug.WriteLine($"\n");
}
}
[TestInitialize]
public void Init()
{
@ -114,5 +282,26 @@ namespace Remotely.Tests
ServiceContainer.Instance = serviceCollection.BuildServiceProvider();
}
private Bitmap GetFrame(string frameFileName)
{
using (var mrs = Assembly.GetExecutingAssembly().GetManifestResourceStream($"Remotely.Tests.Resources.{frameFileName}.jpg"))
{
var resourceImage = (Bitmap)Bitmap.FromStream(mrs);
if (resourceImage.PixelFormat != PixelFormat.Format32bppArgb)
{
return resourceImage.Clone(new Rectangle(0, 0, resourceImage.Width, resourceImage.Height), PixelFormat.Format32bppArgb);
}
return resourceImage;
}
}
private ImageCodecInfo GetEncoder(ImageFormat format)
{
var codecs = ImageCodecInfo.GetImageEncoders();
return codecs.FirstOrDefault(x => x.FormatID == format.Guid);
}
}
}

BIN
Tests/Resources/Frame1.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 444 KiB

BIN
Tests/Resources/Frame2.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 370 KiB

10
Tests/Tests.csproj
View File

@ -12,6 +12,16 @@
<Platforms>AnyCPU;x64;x86</Platforms>
</PropertyGroup>
<ItemGroup>
<None Remove="Resources\Frame1.jpg" />
<None Remove="Resources\Frame2.jpg" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Resources\Frame1.jpg" />
<EmbeddedResource Include="Resources\Frame2.jpg" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="5.0.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="5.0.1" />