mirrors/Remotely
3
0
Fork 0
mirror of https://github.com/immense/Remotely.git synced 2025-10-26 11:27:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace System.Drawing with SkiaSharp.

Browse Source
This commit is contained in:
Jared Goodwin 2022-07-06 07:36:38 -07:00
parent 72d42d7351
commit 47378133d1
15 changed files with 607 additions and 391 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Desktop.Core/Desktop.Core.csproj
View File

@ -46,8 +46,10 @@
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="6.0.0" />
<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="2.2.0" />
<PackageReference Include="Microsoft.MixedReality.WebRTC" Version="2.0.2" />
<PackageReference Include="System.Drawing.Common" Version="6.0.0" />
<PackageReference Include="SkiaSharp" Version="2.88.0" />
<PackageReference Include="SkiaSharp.Views.Desktop.Common" Version="2.88.0" />
<PackageReference Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
</ItemGroup>

66
Desktop.Core/Extensions/SKBitmapExtensions.cs Normal file
View File

@ -0,0 +1,66 @@
using Remotely.Shared;
using SkiaSharp;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Remotely.Desktop.Core.Extensions
{
public static class SKBitmapExtensions
{
public static SKRect ToRectangle(this SKBitmap bitmap)
{
return new SKRect(0, 0, bitmap.Width, bitmap.Height);
}
// The SKBitmap.Copy method has a memory leak somewhere, so I created this.
public static SKBitmap CopyEx(this SKBitmap bitmap, bool disposeOriginal = false)
{
var width = bitmap.Width;
var height = bitmap.Height;
var newBitmap = new SKBitmap(width, height);
var bytesPerPixel = bitmap.BytesPerPixel;
var totalSize = bitmap.ByteCount;
unsafe
{
byte* scan1 = (byte*)bitmap.GetPixels().ToPointer();
byte* scan2 = (byte*)newBitmap.GetPixels().ToPointer();
for (var row = 0; row < height; row++)
{
for (var column = 0; column < width; column++)
{
var index = (row * width * bytesPerPixel) + (column * bytesPerPixel);
byte* data1 = scan1 + index;
byte* data2 = scan2 + index;
if (data1[0] != data2[0] ||
data1[1] != data2[1] ||
data1[2] != data2[2] ||
data1[3] != data2[3])
{
data2[0] = data2[0];
data2[1] = data2[1];
data2[2] = data2[2];
data2[3] = data2[3];
}
}
}
}
if (disposeOriginal)
{
bitmap.Dispose();
}
return newBitmap;
}
}
}

9
Desktop.Core/Interfaces/IScreenCapturer.cs
View File

@ -1,4 +1,6 @@
using System;
using Remotely.Shared;
using SkiaSharp;
using System;
using System.Collections.Generic;
using System.Drawing;
@ -13,8 +15,11 @@ namespace Remotely.Desktop.Core.Interfaces
string SelectedScreen { get; }
IEnumerable<string> GetDisplayNames();
SKRect GetFrameDiffArea();
Bitmap GetNextFrame();
Result<SKBitmap> GetImageDiff();
SKBitmap GetNextFrame();
int GetScreenCount();

68
Desktop.Core/Services/ScreenCaster.cs
View File

@ -15,6 +15,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.ComponentModel.DataAnnotations;
using SkiaSharp;
namespace Remotely.Desktop.Core.Services
{
@ -51,10 +52,7 @@ namespace Remotely.Desktop.Core.Services
{
try
{
Bitmap currentFrame = null;
Bitmap previousFrame = null;
long sequence = 0;
var viewer = ServiceContainer.Instance.GetRequiredService<Viewer>();
viewer.Name = screenCastRequest.RequesterName;
viewer.ViewerConnectionID = screenCastRequest.ViewerID;
@ -84,8 +82,6 @@ namespace Remotely.Desktop.Core.Services
screenBounds.Width,
screenBounds.Height);
await viewer.SendScreenSize(screenBounds.Width, screenBounds.Height);
await viewer.SendCursorChange(_cursorIconWatcher.GetCurrentCursor());
await viewer.SendWindowsSessions();
@ -95,20 +91,19 @@ namespace Remotely.Desktop.Core.Services
await viewer.SendScreenSize(bounds.Width, bounds.Height);
};
using (var initialFrame = viewer.Capturer.GetNextFrame())
// This gets disposed internally in the Capturer on the next call.
var initialFrame = viewer.Capturer.GetNextFrame();
if (initialFrame != null)
{
if (initialFrame != null)
await viewer.SendScreenCapture(new CaptureFrame()
{
await viewer.SendScreenCapture(new CaptureFrame()
{
EncodedImageBytes = ImageUtils.EncodeJpeg(initialFrame),
Left = screenBounds.Left,
Top = screenBounds.Top,
Width = screenBounds.Width,
Height = screenBounds.Height,
Sequence = sequence++
});
}
EncodedImageBytes = ImageUtils.EncodeBitmap(initialFrame, SKEncodedImageFormat.Jpeg, viewer.ImageQuality),
Left = screenBounds.Left,
Top = screenBounds.Top,
Width = screenBounds.Width,
Height = screenBounds.Height
});
}
@ -147,21 +142,9 @@ namespace Remotely.Desktop.Core.Services
viewer.ApplyAutoQuality();
if (currentFrame != null)
{
previousFrame?.Dispose();
previousFrame = (Bitmap)currentFrame.Clone();
}
var currentFrame = viewer.Capturer.GetNextFrame();
currentFrame?.Dispose();
currentFrame = viewer.Capturer.GetNextFrame();
if (currentFrame is null)
{
continue;
}
var diffArea = ImageUtils.GetDiffArea(currentFrame, previousFrame, viewer.Capturer.CaptureFullscreen);
var diffArea = viewer.Capturer.GetFrameDiffArea();
if (diffArea.IsEmpty)
{
@ -170,18 +153,9 @@ namespace Remotely.Desktop.Core.Services
viewer.Capturer.CaptureFullscreen = false;
using var croppedFrame = currentFrame.Clone(diffArea, currentFrame.PixelFormat);
using var croppedFrame = ImageUtils.CropBitmap(currentFrame, diffArea);
byte[] encodedImageBytes;
if (viewer.ImageQuality == Viewer.DefaultQuality)
{
encodedImageBytes = ImageUtils.EncodeJpeg(croppedFrame);
}
else
{
encodedImageBytes = ImageUtils.EncodeJpeg(croppedFrame, viewer.ImageQuality);
}
var encodedImageBytes = ImageUtils.EncodeBitmap(croppedFrame, SKEncodedImageFormat.Jpeg, 80);
await SendFrame(encodedImageBytes, diffArea, sequence++, viewer);
@ -217,7 +191,7 @@ namespace Remotely.Desktop.Core.Services
}
}
private static async Task SendFrame(byte[] encodedImageBytes, Rectangle diffArea, long sequence, Viewer viewer)
private static async Task SendFrame(byte[] encodedImageBytes, SKRect diffArea, long sequence, Viewer viewer)
{
if (encodedImageBytes.Length == 0)
{
@ -227,10 +201,10 @@ namespace Remotely.Desktop.Core.Services
await viewer.SendScreenCapture(new CaptureFrame()
{
EncodedImageBytes = encodedImageBytes,
Top = diffArea.Top,
Left = diffArea.Left,
Width = diffArea.Width,
Height = diffArea.Height,
Top = (int)diffArea.Top,
Left = (int)diffArea.Left,
Width = (int)diffArea.Width,
Height = (int)diffArea.Height,
Sequence = sequence
});
}

54
Desktop.Core/Services/Viewer.cs
View File

@ -279,17 +279,22 @@ namespace Remotely.Desktop.Core.Services
var width = screenFrame.Width;
var height = screenFrame.Height;
for (var i = 0; i < screenFrame.EncodedImageBytes.Length; i += 50_000)
var chunks = screenFrame.EncodedImageBytes.Chunk(50_000).ToArray();
var chunkCount = chunks.Length;
for (var i = 0; i < chunkCount; i++)
{
var chunk = chunks[i];
var dto = new CaptureFrameDto()
{
Left = left,
Top = top,
Width = width,
Height = height,
EndOfFrame = false,
EndOfFrame = i == chunkCount - 1,
Sequence = screenFrame.Sequence,
ImageBytes = screenFrame.EncodedImageBytes.Skip(i).Take(50_000).ToArray()
ImageBytes = chunk
};
await SendToViewer(
@ -297,19 +302,38 @@ namespace Remotely.Desktop.Core.Services
() => CasterSocket.SendDtoToViewer(dto, ViewerConnectionID));
}
var endOfFrameDto = new CaptureFrameDto()
{
Left = left,
Top = top,
Width = width,
Height = height,
EndOfFrame = true,
Sequence = screenFrame.Sequence,
};
await SendToViewer(
() => RtcSession.SendDto(endOfFrameDto),
() => CasterSocket.SendDtoToViewer(endOfFrameDto, ViewerConnectionID));
//for (var i = 0; i < screenFrame.EncodedImageBytes.Length; i += 50_000)
//{
// var dto = new CaptureFrameDto()
// {
// Left = left,
// Top = top,
// Width = width,
// Height = height,
// EndOfFrame = false,
// Sequence = screenFrame.Sequence,
// ImageBytes = screenFrame.EncodedImageBytes.Skip(i).Take(50_000).ToArray()
// };
// await SendToViewer(
// () => RtcSession.SendDto(dto),
// () => CasterSocket.SendDtoToViewer(dto, ViewerConnectionID));
//}
//var endOfFrameDto = new CaptureFrameDto()
//{
// Left = left,
// Top = top,
// Width = width,
// Height = height,
// EndOfFrame = true,
// Sequence = screenFrame.Sequence,
//};
//await SendToViewer(
// () => RtcSession.SendDto(endOfFrameDto),
// () => CasterSocket.SendDtoToViewer(endOfFrameDto, ViewerConnectionID));
}
public async Task SendScreenData(

26
Desktop.Core/Services/WebRtcSession.cs
View File

@ -182,26 +182,16 @@ namespace Remotely.Desktop.Core.Services
return;
}
var bitmapData = currentFrame.LockBits(
new Rectangle(Point.Empty, Viewer.Capturer.CurrentScreenBounds.Size),
System.Drawing.Imaging.ImageLockMode.ReadOnly,
System.Drawing.Imaging.PixelFormat.Format32bppArgb);
var pixels = currentFrame.GetPixels();
try
var frame = new Argb32VideoFrame()
{
var frame = new Argb32VideoFrame()
{
data = bitmapData.Scan0,
height = (uint)currentFrame.Height,
width = (uint)currentFrame.Width,
stride = bitmapData.Stride
};
request.CompleteRequest(in frame);
}
finally
{
currentFrame.UnlockBits(bitmapData);
}
data = pixels,
height = (uint)currentFrame.Height,
width = (uint)currentFrame.Width,
stride = currentFrame.RowBytes
};
request.CompleteRequest(in frame);
}
catch (Exception ex)
{

282
Desktop.Core/Utilities/ImageUtils.cs
View File

@ -1,4 +1,10 @@
using System;
using Microsoft.IO;
using Remotely.Desktop.Core.Extensions;
using Remotely.Shared;
using Remotely.Shared.Utilities;
using SkiaSharp;
using SkiaSharp.Views.Desktop;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Drawing;
@ -12,82 +18,139 @@ namespace Remotely.Desktop.Core.Utilities
{
public class ImageUtils
{
private static readonly RecyclableMemoryStreamManager _recycleManager = new();
private static readonly ImageCodecInfo _jpegEncoder = ImageCodecInfo.GetImageEncoders().FirstOrDefault(x => x.FormatID == ImageFormat.Jpeg.Guid);
//public static byte[] EncodeWithSkia(Bitmap bitmap, SKEncodedImageFormat format, int quality)
//{
// using var ms = new MemoryStream();
// var info = new SKImageInfo(bitmap.Width, bitmap.Height);
// var skBitmap = new SKBitmap(info);
// using (var pixmap = skBitmap.PeekPixels())
// {
// bitmap.ToSKPixmap(pixmap);
// }
// skBitmap.Encode(ms, format, quality);
// return ms.ToArray();
//}
public static byte[] EncodeJpeg(Bitmap bitmap)
public static byte[] EncodeBitmap(SKBitmap bitmap, SKEncodedImageFormat format, int quality)
{
using var ms = new MemoryStream();
bitmap.Save(ms, ImageFormat.Jpeg);
using var ms = _recycleManager.GetStream();
bitmap.Encode(ms, format, quality);
return ms.ToArray();
}
public static byte[] EncodeJpeg(Bitmap bitmap, int quality)
public static SKBitmap CropBitmap(SKBitmap bitmap, SKRect cropArea)
{
using var ms = new MemoryStream();
using var encoderParams = new EncoderParameters(1)
{
Param = new[] { new EncoderParameter(Encoder.Quality, quality) }
};
bitmap.Save(ms, _jpegEncoder, encoderParams);
return ms.ToArray();
var cropped = new SKBitmap((int)cropArea.Width, (int)cropArea.Height);
using var canvas = new SKCanvas(cropped);
canvas.DrawBitmap(
bitmap,
cropArea,
new SKRect(0, 0, cropArea.Width, cropArea.Height));
return cropped;
}
public static Rectangle GetDiffArea(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen)
public static Result<SKBitmap> GetImageDiff(SKBitmap currentFrame, SKBitmap previousFrame, bool forceFullscreen = false)
{
if (currentFrame == null || previousFrame == null)
{
return Rectangle.Empty;
}
if (captureFullscreen)
{
return new Rectangle(new Point(0, 0), currentFrame.Size);
}
if (currentFrame.Height != previousFrame.Height || currentFrame.Width != previousFrame.Width)
{
throw new Exception("Bitmaps are not of equal dimensions.");
}
if (currentFrame.PixelFormat != previousFrame.PixelFormat)
{
throw new Exception("Bitmaps are not the same format.");
}
var width = currentFrame.Width;
var height = currentFrame.Height;
int left = int.MaxValue;
int top = int.MaxValue;
int right = int.MinValue;
int bottom = int.MinValue;
BitmapData bd1 = null;
BitmapData bd2 = null;
try
{
bd1 = previousFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, currentFrame.PixelFormat);
bd2 = currentFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, previousFrame.PixelFormat);
if (currentFrame is null)
{
return Result.Fail<SKBitmap>("Current frame cannot be null.");
}
var bytesPerPixel = Bitmap.GetPixelFormatSize(currentFrame.PixelFormat) / 8;
var totalSize = bd1.Height * bd1.Width * bytesPerPixel;
if (previousFrame is null || forceFullscreen)
{
return Result.Ok(currentFrame.Copy());
}
if (currentFrame.Height != previousFrame.Height ||
currentFrame.Width != previousFrame.Width ||
currentFrame.BytesPerPixel != previousFrame.BytesPerPixel)
{
return Result.Fail<SKBitmap>("Frames are not of equal size.");
}
var width = currentFrame.Width;
var height = currentFrame.Height;
var anyChanges = false;
var diffFrame = new SKBitmap(width, height);
var bytesPerPixel = currentFrame.BytesPerPixel;
var totalSize = currentFrame.ByteCount;
unsafe
{
byte* scan1 = (byte*)bd1.Scan0.ToPointer();
byte* scan2 = (byte*)bd2.Scan0.ToPointer();
byte* scan1 = (byte*)currentFrame.GetPixels().ToPointer();
byte* scan2 = (byte*)previousFrame.GetPixels().ToPointer();
byte* scan3 = (byte*)diffFrame.GetPixels().ToPointer();
for (var row = 0; row < height; row++)
{
for (var column = 0; column < width; column++)
{
var index = (row * width * bytesPerPixel) + (column * bytesPerPixel);
byte* data1 = scan1 + index;
byte* data2 = scan2 + index;
byte* data3 = scan3 + index;
if (data1[0] != data2[0] ||
data1[1] != data2[1] ||
data1[2] != data2[2] ||
data1[3] != data2[3])
{
anyChanges = true;
data3[0] = data2[0];
data3[1] = data2[1];
data3[2] = data2[2];
data3[3] = data2[3];
}
}
}
}
if (anyChanges)
{
return Result.Ok(diffFrame);
}
diffFrame.Dispose();
return Result.Fail<SKBitmap>("No difference found.");
}
catch (Exception ex)
{
Logger.Write(ex, "Error while getting image diff.");
return Result.Fail<SKBitmap>(ex);
}
}
public static SKRect GetDiffArea(SKBitmap currentFrame, SKBitmap previousFrame, bool forceFullscreen = false)
{
try
{
if (currentFrame is null)
{
return SKRect.Empty;
}
if (previousFrame is null || forceFullscreen)
{
return currentFrame.ToRectangle();
}
if (currentFrame.Height != previousFrame.Height ||
currentFrame.Width != previousFrame.Width ||
currentFrame.BytesPerPixel != previousFrame.BytesPerPixel)
{
return SKRect.Empty;
}
var width = currentFrame.Width;
var height = currentFrame.Height;
int left = int.MaxValue;
int top = int.MaxValue;
int right = int.MinValue;
int bottom = int.MinValue;
var bytesPerPixel = currentFrame.BytesPerPixel;
var totalSize = currentFrame.ByteCount;
unsafe
{
byte* scan1 = (byte*)currentFrame.GetPixels().ToPointer();
byte* scan2 = (byte*)previousFrame.GetPixels().ToPointer();
for (var row = 0; row < height; row++)
{
@ -124,106 +187,23 @@ namespace Remotely.Desktop.Core.Utilities
}
}
// Check for valid bounding box.
if (left <= right && top <= bottom)
{
// Bounding box is valid. Padding is necessary to prevent artifacts from
// moving windows.
left = Math.Max(left - 2, 0);
top = Math.Max(top - 2, 0);
right = Math.Min(right + 2, width);
bottom = Math.Min(bottom + 2, height);
return new SKRect(left, top, right, bottom);
}
return new Rectangle(left, top, right - left, bottom - top);
}
else
{
return Rectangle.Empty;
}
return SKRect.Empty;
}
}
catch
catch (Exception ex)
{
return Rectangle.Empty;
}
finally
{
currentFrame.UnlockBits(bd1);
previousFrame.UnlockBits(bd2);
}
}
public static Bitmap GetImageDiff(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen, out bool hadChanges)
{
hadChanges = false;
if (currentFrame is null || previousFrame is null)
{
hadChanges = false;
return null;
}
if (captureFullscreen)
{
hadChanges = true;
return (Bitmap)currentFrame.Clone();
}
if (currentFrame.Height != previousFrame.Height || currentFrame.Width != previousFrame.Width)
{
throw new Exception("Bitmaps are not of equal dimensions.");
}
var width = currentFrame.Width;
var height = currentFrame.Height;
var mergedFrame = new Bitmap(width, height);
var bd1 = previousFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, currentFrame.PixelFormat);
var bd2 = currentFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, previousFrame.PixelFormat);
var bd3 = mergedFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.WriteOnly, currentFrame.PixelFormat);
try
{
var bytesPerPixel = Bitmap.GetPixelFormatSize(currentFrame.PixelFormat) / 8;
var totalSize = bd1.Height * bd1.Width * bytesPerPixel;
unsafe
{
byte* scan1 = (byte*)bd1.Scan0.ToPointer();
byte* scan2 = (byte*)bd2.Scan0.ToPointer();
byte* scan3 = (byte*)bd3.Scan0.ToPointer();
for (int counter = 0; counter < totalSize - bytesPerPixel; counter += bytesPerPixel)
{
byte* data1 = scan1 + counter;
byte* data2 = scan2 + counter;
byte* data3 = scan3 + counter;
if (data1[0] != data2[0] ||
data1[1] != data2[1] ||
data1[2] != data2[2] ||
data1[3] != data2[3])
{
hadChanges = true;
data3[0] = data2[0];
data3[1] = data2[1];
data3[2] = data2[2];
data3[3] = data2[3];
}
}
}
return mergedFrame;
}
catch
{
return mergedFrame;
}
finally
{
previousFrame.UnlockBits(bd1);
currentFrame.UnlockBits(bd2);
mergedFrame.UnlockBits(bd3);
Logger.Write(ex, "Error while getting area diff.");
return SKRect.Empty;
}
}
}

2
Desktop.Win/Desktop.Win.csproj
View File

@ -17,6 +17,7 @@
<PackageProjectUrl>https://remotely.one</PackageProjectUrl>
<Platforms>AnyCPU;x86;x64</Platforms>
<StartupObject>Remotely.Desktop.Win.App</StartupObject>
<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@ -42,7 +43,6 @@
<PackageReference Include="SharpDX" Version="4.2.0" />
<PackageReference Include="SharpDX.Direct3D11" Version="4.2.0" />
<PackageReference Include="SharpDX.DXGI" Version="4.2.0" />
<PackageReference Include="System.Drawing.Common" Version="6.0.0" />
</ItemGroup>
<ItemGroup>

238
Desktop.Win/Services/ScreenCapturerWin.cs
View File

@ -23,6 +23,7 @@
using Microsoft.Win32;
using Remotely.Desktop.Core.Interfaces;
using Remotely.Desktop.Core.Utilities;
using Remotely.Desktop.Win.Models;
using Remotely.Shared.Utilities;
using Remotely.Shared.Win32;
@ -34,9 +35,13 @@ using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using System.Windows.Forms;
using Remotely.Shared;
using Result = Remotely.Shared.Result;
using SkiaSharp;
using SkiaSharp.Views.Desktop;
using Remotely.Desktop.Core.Extensions;
namespace Remotely.Desktop.Win.Services
{
@ -45,6 +50,8 @@ namespace Remotely.Desktop.Win.Services
private readonly Dictionary<string, int> _bitBltScreens = new();
private readonly Dictionary<string, DirectXOutput> _directxScreens = new();
private readonly object _screenBoundsLock = new();
private SKBitmap _currentFrame;
private SKBitmap _previousFrame;
public ScreenCapturerWin()
{
@ -54,14 +61,8 @@ namespace Remotely.Desktop.Win.Services
public event EventHandler<Rectangle> ScreenChanged;
private enum GetDirectXFrameResult
{
Success,
Failure,
Timeout,
}
public bool CaptureFullscreen { get; set; } = true;
public Rectangle CurrentScreenBounds { get; private set; } = Screen.PrimaryScreen.Bounds;
public bool NeedsInit { get; set; } = true;
public string SelectedScreen { get; private set; } = Screen.PrimaryScreen.DeviceName;
public void Dispose()
@ -74,11 +75,23 @@ namespace Remotely.Desktop.Win.Services
}
catch { }
}
public Rectangle CurrentScreenBounds { get; private set; } = Screen.PrimaryScreen.Bounds;
public IEnumerable<string> GetDisplayNames()
{
return Screen.AllScreens.Select(x => x.DeviceName);
}
public IEnumerable<string> GetDisplayNames() => Screen.AllScreens.Select(x => x.DeviceName);
public SKRect GetFrameDiffArea()
{
return ImageUtils.GetDiffArea(_currentFrame, _previousFrame, CaptureFullscreen);
}
public Bitmap GetNextFrame()
public Result<SKBitmap> GetImageDiff()
{
return ImageUtils.GetImageDiff(_currentFrame, _previousFrame);
}
public SKBitmap GetNextFrame()
{
lock (_screenBoundsLock)
{
@ -92,6 +105,8 @@ namespace Remotely.Desktop.Win.Services
Init();
}
SwapFrames();
// Sometimes DX will result in a timeout, even when there are changes
// on the screen. I've observed this when a laptop lid is closed, or
// on some machines that aren't connected to a monitor. This will
@ -100,21 +115,17 @@ namespace Remotely.Desktop.Win.Services
if (_directxScreens.TryGetValue(SelectedScreen, out var dxDisplay) &&
dxDisplay.Rotation == DisplayModeRotation.Identity)
{
var (result, frame) = GetDirectXFrame();
var result = GetDirectXFrame();
if (result == GetDirectXFrameResult.Timeout)
if (result.IsSuccess && !IsEmpty(result.Value))
{
return null;
}
if (result == GetDirectXFrameResult.Success)
{
return frame;
_currentFrame = result.Value;
return _currentFrame;
}
}
return GetBitBltFrame();
_currentFrame = GetBitBltFrame();
return _currentFrame;
}
catch (Exception e)
{
@ -127,7 +138,10 @@ namespace Remotely.Desktop.Win.Services
}
public int GetScreenCount() => Screen.AllScreens.Length;
public int GetScreenCount()
{
return Screen.AllScreens.Length;
}
public int GetSelectedScreenIndex()
{
@ -138,7 +152,10 @@ namespace Remotely.Desktop.Win.Services
return 0;
}
public Rectangle GetVirtualScreenBounds() => SystemInformation.VirtualScreen;
public Rectangle GetVirtualScreenBounds()
{
return SystemInformation.VirtualScreen;
}
public void Init()
{
@ -187,16 +204,16 @@ namespace Remotely.Desktop.Win.Services
_directxScreens.Clear();
}
private Bitmap GetBitBltFrame()
private SKBitmap GetBitBltFrame()
{
try
{
var currentFrame = new Bitmap(CurrentScreenBounds.Width, CurrentScreenBounds.Height, PixelFormat.Format32bppArgb);
using var currentFrame = new Bitmap(CurrentScreenBounds.Width, CurrentScreenBounds.Height, PixelFormat.Format32bppArgb);
using (var graphic = Graphics.FromImage(currentFrame))
{
graphic.CopyFromScreen(CurrentScreenBounds.Left, CurrentScreenBounds.Top, 0, 0, new Size(CurrentScreenBounds.Width, CurrentScreenBounds.Height));
}
return currentFrame;
return currentFrame.ToSKBitmap();
}
catch (Exception ex)
{
@ -207,107 +224,86 @@ namespace Remotely.Desktop.Win.Services
return null;
}
private (GetDirectXFrameResult result, Bitmap frame) GetDirectXFrame()
private Result<SKBitmap> GetDirectXFrame()
{
if (!_directxScreens.TryGetValue(SelectedScreen, out var dxOutput))
{
return Result.Fail<SKBitmap>("DirectX output not found.");
}
try
{
var duplicatedOutput = _directxScreens[SelectedScreen].OutputDuplication;
var device = _directxScreens[SelectedScreen].Device;
var texture2D = _directxScreens[SelectedScreen].Texture2D;
var outputDuplication = dxOutput.OutputDuplication;
var device = dxOutput.Device;
var texture2D = dxOutput.Texture2D;
var bounds = new Rectangle(0, 0, texture2D.Description.Width, texture2D.Description.Height);
// Try to get duplicated frame within given time is ms
var result = duplicatedOutput.TryAcquireNextFrame(500,
out var duplicateFrameInformation,
out var screenResource);
var result = outputDuplication.TryAcquireNextFrame(100, out var duplicateFrameInfo, out var screenResource);
if (result.Failure)
if (!result.Success)
{
if (result.Code == SharpDX.DXGI.ResultCode.WaitTimeout.Code)
{
return (GetDirectXFrameResult.Timeout, null);
}
else
{
Logger.Write($"TryAcquireFrame error. Code: {result.Code}");
NeedsInit = true;
return (GetDirectXFrameResult.Failure, null);
}
return Result.Fail<SKBitmap>("Next frame did not arrive.");
}
if (duplicateFrameInformation.AccumulatedFrames == 0)
if (duplicateFrameInfo.AccumulatedFrames == 0)
{
try
{
duplicatedOutput.ReleaseFrame();
outputDuplication.ReleaseFrame();
}
catch { }
return (GetDirectXFrameResult.Failure, null);
return Result.Fail<SKBitmap>("No frames were accumulated.");
}
var currentFrame = new Bitmap(texture2D.Description.Width, texture2D.Description.Height, PixelFormat.Format32bppArgb);
// Copy resource into memory that can be accessed by the CPU
using (var screenTexture2D = screenResource.QueryInterface<Texture2D>())
using Texture2D screenTexture2D = screenResource.QueryInterface<Texture2D>();
device.ImmediateContext.CopyResource(screenTexture2D, texture2D);
var dataBox = device.ImmediateContext.MapSubresource(texture2D, 0, MapMode.Read, SharpDX.Direct3D11.MapFlags.None);
using var bitmap = new Bitmap(bounds.Width, bounds.Height, PixelFormat.Format32bppArgb);
var bitmapData = bitmap.LockBits(bounds, ImageLockMode.WriteOnly, bitmap.PixelFormat);
var dataBoxPointer = dataBox.DataPointer;
var bitmapDataPointer = bitmapData.Scan0;
for (var y = 0; y < bounds.Height; y++)
{
device.ImmediateContext.CopyResource(screenTexture2D, texture2D);
SharpDX.Utilities.CopyMemory(bitmapDataPointer, dataBoxPointer, bounds.Width * 4);
dataBoxPointer = IntPtr.Add(dataBoxPointer, dataBox.RowPitch);
bitmapDataPointer = IntPtr.Add(bitmapDataPointer, bitmapData.Stride);
}
// Get the desktop capture texture
var mapSource = device.ImmediateContext.MapSubresource(texture2D, 0, MapMode.Read, SharpDX.Direct3D11.MapFlags.None);
var boundsRect = new Rectangle(0, 0, texture2D.Description.Width, texture2D.Description.Height);
// Copy pixels from screen capture Texture to GDI bitmap
var mapDest = currentFrame.LockBits(boundsRect, ImageLockMode.WriteOnly, currentFrame.PixelFormat);
var sourcePtr = mapSource.DataPointer;
var destPtr = mapDest.Scan0;
for (int y = 0; y < texture2D.Description.Height; y++)
{
// Copy a single line
SharpDX.Utilities.CopyMemory(destPtr, sourcePtr, texture2D.Description.Width * 4);
// Advance pointers
sourcePtr = IntPtr.Add(sourcePtr, mapSource.RowPitch);
destPtr = IntPtr.Add(destPtr, mapDest.Stride);
}
// Release source and dest locks
currentFrame.UnlockBits(mapDest);
bitmap.UnlockBits(bitmapData);
device.ImmediateContext.UnmapSubresource(texture2D, 0);
screenResource.Dispose();
duplicatedOutput.ReleaseFrame();
return (GetDirectXFrameResult.Success, currentFrame);
screenResource?.Dispose();
return Result.Ok(bitmap.ToSKBitmap());
}
catch (SharpDXException e)
{
if (e.ResultCode.Code == SharpDX.DXGI.ResultCode.WaitTimeout.Code)
if (e.ResultCode.Code == SharpDX.DXGI.ResultCode.WaitTimeout.Result.Code)
{
return (GetDirectXFrameResult.Timeout, null);
return Result.Fail<SKBitmap>("DirectX timed out while waiting for frame.");
}
Logger.Write(e, "SharpDXException error.");
Logger.Write(e);
}
catch (Exception ex)
{
Logger.Write(ex);
Logger.Write(ex, "Error while grabbing with DirectX.");
}
return (GetDirectXFrameResult.Failure, null);
finally
{
try
{
dxOutput.OutputDuplication.ReleaseFrame();
}
catch { }
}
return Result.Fail<SKBitmap>("Failed to get DirectX grab.");
}
private void InitBitBlt()
{
try
_bitBltScreens.Clear();
for (var i = 0; i < Screen.AllScreens.Length; i++)
{
_bitBltScreens.Clear();
for (var i = 0; i < Screen.AllScreens.Length; i++)
{
_bitBltScreens.Add(Screen.AllScreens[i].DeviceName, i);
}
}
catch (Exception ex)
{
Logger.Write(ex);
_bitBltScreens.Add(Screen.AllScreens[i].DeviceName, i);
}
}
@ -369,6 +365,50 @@ namespace Remotely.Desktop.Win.Services
}
}
private bool IsEmpty(SKBitmap bitmap)
{
if (bitmap is null)
{
return true;
}
var height = bitmap.Height;
var width = bitmap.Width;
var bytesPerPixel = bitmap.BytesPerPixel;
try
{
unsafe
{
byte* scan = (byte*)bitmap.GetPixels();
for (var row = 0; row < height; row++)
{
for (var column = 0; column < width; column++)
{
var index = (row * width * bytesPerPixel) + (column * bytesPerPixel);
byte* data = scan + index;
for (var i = 0; i < bytesPerPixel; i++)
{
if (data[i] != 0)
{
return false;
}
}
}
}
return true;
}
}
catch
{
return true;
}
}
private void RefreshCurrentScreenBounds()
{
CurrentScreenBounds = Screen.AllScreens[_bitBltScreens[SelectedScreen]].Bounds;
@ -377,6 +417,14 @@ namespace Remotely.Desktop.Win.Services
ScreenChanged?.Invoke(this, CurrentScreenBounds);
}
private void SwapFrames()
{
if (_currentFrame != null)
{
_previousFrame?.Dispose();
_previousFrame = _currentFrame;
}
}
private void SystemEvents_DisplaySettingsChanged(object sender, EventArgs e)
{
RefreshCurrentScreenBounds();

66
Desktop.XPlat/Services/ScreenCapturerLinux.cs
View File

@ -1,14 +1,15 @@
using Remotely.Desktop.Core.Interfaces;
using Remotely.Desktop.Core.Utilities;
using Remotely.Desktop.XPlat.Native.Linux;
using Remotely.Shared;
using Remotely.Shared.Utilities;
using SkiaSharp;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text.Json;
using System.Threading;
namespace Remotely.Desktop.XPlat.Services
{
@ -16,6 +17,9 @@ namespace Remotely.Desktop.XPlat.Services
{
private readonly object _screenBoundsLock = new();
private readonly Dictionary<string, LibXrandr.XRRMonitorInfo> _x11Screens = new();
private SKBitmap _currentFrame;
private SKBitmap _previousFrame;
public ScreenCapturerLinux()
{
Display = LibX11.XOpenDisplay(null);
@ -34,15 +38,42 @@ namespace Remotely.Desktop.XPlat.Services
LibX11.XCloseDisplay(Display);
GC.SuppressFinalize(this);
}
public IEnumerable<string> GetDisplayNames() => _x11Screens.Keys.Select(x => x.ToString());
public IEnumerable<string> GetDisplayNames()
{
return _x11Screens.Keys.Select(x => x.ToString());
}
public Bitmap GetNextFrame()
public SKRect GetFrameDiffArea()
{
return ImageUtils.GetDiffArea(_currentFrame, _previousFrame, CaptureFullscreen);
}
public Result<SKBitmap> GetImageDiff()
{
return ImageUtils.GetImageDiff(_currentFrame, _previousFrame);
}
public SKBitmap GetNextFrame()
{
lock (_screenBoundsLock)
{
try
{
return GetX11Capture();
if (_currentFrame != null)
{
_previousFrame?.Dispose();
try
{
_previousFrame = _currentFrame;
}
catch (Exception ex)
{
Logger.Write(ex);
}
}
_currentFrame = GetX11Capture();
return _currentFrame;
}
catch (Exception ex)
{
@ -52,9 +83,15 @@ namespace Remotely.Desktop.XPlat.Services
}
}
}
public int GetScreenCount() => _x11Screens.Count;
public int GetScreenCount()
{
return _x11Screens.Count;
}
public int GetSelectedScreenIndex() => int.Parse(SelectedScreen ?? "0");
public int GetSelectedScreenIndex()
{
return int.Parse(SelectedScreen ?? "0");
}
public Rectangle GetVirtualScreenBounds()
{
@ -77,7 +114,7 @@ namespace Remotely.Desktop.XPlat.Services
{
CaptureFullscreen = true;
_x11Screens.Clear();
var monitorsPtr = LibXrandr.XRRGetMonitors(Display, LibX11.XDefaultRootWindow(Display), true, out var monitorCount);
var monitorInfoSize = Marshal.SizeOf<LibXrandr.XRRMonitorInfo>();
@ -140,9 +177,9 @@ namespace Remotely.Desktop.XPlat.Services
}
}
private Bitmap GetX11Capture()
private SKBitmap GetX11Capture()
{
var currentFrame = new Bitmap(CurrentScreenBounds.Width, CurrentScreenBounds.Height, PixelFormat.Format32bppArgb);
var currentFrame = new SKBitmap(CurrentScreenBounds.Width, CurrentScreenBounds.Height);
var window = LibX11.XDefaultRootWindow(Display);
@ -157,20 +194,19 @@ namespace Remotely.Desktop.XPlat.Services
var image = Marshal.PtrToStructure<LibX11.XImage>(imagePointer);
var bd = currentFrame.LockBits(new Rectangle(0, 0, CurrentScreenBounds.Width, CurrentScreenBounds.Height), ImageLockMode.WriteOnly, PixelFormat.Format32bppArgb);
var pixels = currentFrame.GetPixels();
unsafe
{
byte* scan1 = (byte*)bd.Scan0.ToPointer();
byte* scan1 = (byte*)pixels.ToPointer();
byte* scan2 = (byte*)image.data.ToPointer();
var bytesPerPixel = Bitmap.GetPixelFormatSize(currentFrame.PixelFormat) / 8;
var totalSize = bd.Height * bd.Width * bytesPerPixel;
var bytesPerPixel = currentFrame.BytesPerPixel;
var totalSize = currentFrame.Height * currentFrame.Width * bytesPerPixel;
for (int counter = 0; counter < totalSize - bytesPerPixel; counter++)
{
scan1[counter] = scan2[counter];
}
}
currentFrame.UnlockBits(bd);
Marshal.DestroyStructure<LibX11.XImage>(imagePointer);
LibX11.XDestroyImage(imagePointer);

11
Server/wwwroot/src/RemoteControl/CaptureProcessor.js
View File

@ -5,7 +5,10 @@ const ImagePartials = {};
let CanvasLock = 1;
let NextSequence = 0;
export function HandleCaptureReceived(captureFrame) {
// TODO: Move to OffscreenCanvas after it's fully supported.
if (!ImagePartials[captureFrame.Sequence]) {
ImagePartials[captureFrame.Sequence] = [];
}
ImagePartials[captureFrame.Sequence].push(captureFrame.ImageBytes);
if (captureFrame.EndOfFrame) {
var partials = ImagePartials[captureFrame.Sequence];
let completedFrame = new Blob(partials);
@ -32,12 +35,6 @@ export function HandleCaptureReceived(captureFrame) {
// window.URL.revokeObjectURL(url);
//};
}
else {
if (!ImagePartials[captureFrame.Sequence]) {
ImagePartials[captureFrame.Sequence] = [];
}
ImagePartials[captureFrame.Sequence].push(captureFrame.ImageBytes);
}
}
function processQueue() {
try {

2
Server/wwwroot/src/RemoteControl/CaptureProcessor.js.map
View File

@ -1 +1 @@
{"version":3,"file":"CaptureProcessor.js","sourceRoot":"","sources":["CaptureProcessor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAGrC,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAG1C,MAAM,WAAW,GAA0B,EAAE,CAAC;AAC9C,MAAM,aAAa,GAAuC,EAAG,CAAC;AAC9D,IAAI,UAAU,GAAW,CAAC,CAAC;AAC3B,IAAI,YAAY,GAAW,CAAC,CAAC;AAE7B,MAAM,UAAU,qBAAqB,CAAC,YAA6B;IAC/D,4DAA4D;IAC5D,IAAI,YAAY,CAAC,UAAU,EAAE;QAEzB,IAAI,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,cAAc,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,OAAO,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAE5C,iBAAiB,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YAC5C,WAAW,CAAC,IAAI,CAAC;gBACb,YAAY,EAAE,MAAM;gBACpB,SAAS,EAAE,YAAY;aAC1B,CAAC,CAAC;YAEH,IAAI,UAAU,GAAG,CAAC,EAAE;gBAChB,OAAO;aACV;YAED,UAAU,EAAE,CAAC;YAEb,YAAY,EAAE,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,uDAAuD;QACvD,+DAA+D;QAC/D,sBAAsB;QACtB,uCAAuC;QACvC,4BAA4B;QAC5B,2BAA2B;QAC3B,6BAA6B;QAC7B,+BAA+B;QAC/B,sCAAsC;QACtC,IAAI;KACP;SACI;QACD,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE;YACvC,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;SAC7C;QACD,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;KACtE;AACL,CAAC;AAGD,SAAS,YAAY;IACjB,IAAI;QACA,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE;YACxB,IAAI,cAAc,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,OAAO,GAAG,cAAc,CAAC,YAAY,CAAC;YAC1C,IAAI,IAAI,GAAG,cAAc,CAAC,SAAS,CAAC;YAEpC,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,EAAE;gBACpB,YAAY,GAAG,CAAC,CAAC;gBACjB,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;aACzB;YAED,IAAI,IAAI,CAAC,QAAQ,IAAI,YAAY,EAAE;gBAC/B,OAAO,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAA;gBAClE,WAAW,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBACjC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;gBACxE,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC3C,OAAO;aACV;YAED,iBAAiB,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;gBACrC,eAAe,CAAC,SAAS,CAAC,MAAM,EAC5B,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,CAAC,CAAC;gBAEjB,MAAM,CAAC,KAAK,EAAE,CAAC;gBAEf,SAAS,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC;gBAC5C,YAAY,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACjC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAA;SACL;aACI;YACD,UAAU,EAAE,CAAC;SAChB;KACJ;IACD,OAAO,EAAE,EAAE;QACP,UAAU,GAAG,CAAC,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;KACrB;AACL,CAAC"}
{"version":3,"file":"CaptureProcessor.js","sourceRoot":"","sources":["CaptureProcessor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAGrC,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAG1C,MAAM,WAAW,GAA0B,EAAE,CAAC;AAC9C,MAAM,aAAa,GAAuC,EAAG,CAAC;AAC9D,IAAI,UAAU,GAAW,CAAC,CAAC;AAC3B,IAAI,YAAY,GAAW,CAAC,CAAC;AAE7B,MAAM,UAAU,qBAAqB,CAAC,YAA6B;IAC/D,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE;QACvC,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;KAC7C;IAED,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;IAEnE,IAAI,YAAY,CAAC,UAAU,EAAE;QAEzB,IAAI,QAAQ,GAAG,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,cAAc,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,OAAO,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAE5C,iBAAiB,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YAC5C,WAAW,CAAC,IAAI,CAAC;gBACb,YAAY,EAAE,MAAM;gBACpB,SAAS,EAAE,YAAY;aAC1B,CAAC,CAAC;YAEH,IAAI,UAAU,GAAG,CAAC,EAAE;gBAChB,OAAO;aACV;YAED,UAAU,EAAE,CAAC;YAEb,YAAY,EAAE,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,uDAAuD;QACvD,+DAA+D;QAC/D,sBAAsB;QACtB,uCAAuC;QACvC,4BAA4B;QAC5B,2BAA2B;QAC3B,6BAA6B;QAC7B,+BAA+B;QAC/B,sCAAsC;QACtC,IAAI;KACP;AACL,CAAC;AAGD,SAAS,YAAY;IACjB,IAAI;QACA,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE;YACxB,IAAI,cAAc,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,OAAO,GAAG,cAAc,CAAC,YAAY,CAAC;YAC1C,IAAI,IAAI,GAAG,cAAc,CAAC,SAAS,CAAC;YAEpC,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,EAAE;gBACpB,YAAY,GAAG,CAAC,CAAC;gBACjB,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;aACzB;YAED,IAAI,IAAI,CAAC,QAAQ,IAAI,YAAY,EAAE;gBAC/B,OAAO,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAA;gBAClE,WAAW,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBACjC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;gBACxE,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC3C,OAAO;aACV;YAED,iBAAiB,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;gBACrC,eAAe,CAAC,SAAS,CAAC,MAAM,EAC5B,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,MAAM,CAAC,CAAC;gBAEjB,MAAM,CAAC,KAAK,EAAE,CAAC;gBAEf,SAAS,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC;gBAC5C,YAAY,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACjC,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAA;SACL;aACI;YACD,UAAU,EAAE,CAAC;SAChB;KACJ;IACD,OAAO,EAAE,EAAE;QACP,UAAU,GAAG,CAAC,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;KACrB;AACL,CAAC"}

15
Server/wwwroot/src/RemoteControl/CaptureProcessor.ts
View File

@ -10,9 +10,14 @@ let CanvasLock: number = 1;
let NextSequence: number = 0;
export function HandleCaptureReceived(captureFrame: CaptureFrameDto) {
// TODO: Move to OffscreenCanvas after it's fully supported.
if (captureFrame.EndOfFrame) {
if (!ImagePartials[captureFrame.Sequence]) {
ImagePartials[captureFrame.Sequence] = [];
}
ImagePartials[captureFrame.Sequence].push(captureFrame.ImageBytes);
if (captureFrame.EndOfFrame) {
var partials = ImagePartials[captureFrame.Sequence];
let completedFrame = new Blob(partials);
delete ImagePartials[captureFrame.Sequence];
@ -43,12 +48,6 @@ export function HandleCaptureReceived(captureFrame: CaptureFrameDto) {
// window.URL.revokeObjectURL(url);
//};
}
else {
if (!ImagePartials[captureFrame.Sequence]) {
ImagePartials[captureFrame.Sequence] = [];
}
ImagePartials[captureFrame.Sequence].push(captureFrame.ImageBytes);
}
}

81
Shared/Result.cs Normal file
View File

@ -0,0 +1,81 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Remotely.Shared
{
public class Result
{
public static Result<T> Empty<T>()
{
return new Result<T>(true, default);
}
public static Result Fail(string error)
{
return new Result(false, error);
}
public static Result Fail(Exception ex)
{
return new Result(false, null, ex);
}
public static Result<T> Fail<T>(string error)
{
return new Result<T>(false, default, error);
}
public static Result<T> Fail<T>(Exception ex)
{
return new Result<T>(false, default, exception: ex);
}
public static Result Ok()
{
return new Result(true);
}
public static Result<T> Ok<T>(T value)
{
return new Result<T>(true, value, null);
}
public Result(bool isSuccess, string error = null, Exception exception = null)
{
IsSuccess = isSuccess;
Error = error;
Exception = exception;
}
public bool IsSuccess { get; init; }
public string Error { get; init; }
public Exception Exception { get; init; }
}
public class Result<T>
{
public Result(bool isSuccess, T value, string error = null, Exception exception = null)
{
IsSuccess = isSuccess;
Error = error;
Value = value;
Exception = exception;
}
public bool IsSuccess { get; init; }
public string Error { get; init; }
public Exception Exception { get; init; }
public T Value { get; init; }
}
}

74
Tests/ManualTests.cs
View File

@ -9,17 +9,17 @@ using Remotely.Desktop.Core.Utilities;
using Remotely.Desktop.Win.Services;
using Remotely.Shared.Models;
using Remotely.Shared.Models.RemoteControlDtos;
using SkiaSharp;
using SkiaSharp.Views.Desktop;
using System;
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Media.Imaging;
namespace Remotely.Tests
{
@ -39,8 +39,8 @@ namespace Remotely.Tests
private ScreenCaster _screenCaster;
private Mock<ISessionIndicator> _sessionIndicator;
private Mock<IShutdownService> _shutdown;
private Viewer _viewer;
private Mock<IWebRtcSessionFactory> _webrtcFactory;
private Viewer _viewer;
[TestMethod]
@ -68,24 +68,17 @@ namespace Remotely.Tests
_viewer.Dispose();
}
[TestMethod]
[Ignore("Manual test.")]
public void EncodingTests()
{
for (var i = 0; i < 2; i++)
{
var encoderParams = new EncoderParameters()
{
Param = new EncoderParameter[]
{
new EncoderParameter(Encoder.Quality, 60L)
}
};
using var frame1 = GetFrame("Frame1");
using var frame2 = GetFrame("Frame2");
var jpegEncoder = GetEncoder(ImageFormat.Jpeg);
byte[] imageBytes;
var sw = Stopwatch.StartNew();
@ -94,32 +87,49 @@ namespace Remotely.Tests
var diff = ImageUtils.GetDiffArea(frame1, frame2, false);
var diffSize = 0;
using (var tempImage = (Bitmap)frame1.Clone(new Rectangle(diff.X, diff.Y, diff.Width, diff.Height), PixelFormat.Format32bppArgb))
{
var resizeW = diff.Width * 60 / 100;
var resizeH = diff.Height * 60 / 100;
using var resized = new Bitmap(tempImage, new Size(resizeW, resizeH));
using var ms = new MemoryStream();
resized.Save(ms, jpegEncoder, encoderParams);
diffSize = ms.ToArray().Length;
using (var tempImage = ImageUtils.CropBitmap(frame1, diff))
{
imageBytes = ImageUtils.EncodeBitmap(tempImage, SKEncodedImageFormat.Jpeg, 60);
diffSize = imageBytes.Length;
}
Debug.WriteLine($"Diff area size: {diffSize}");
Debug.WriteLine($"Diff area time: {sw.Elapsed.TotalMilliseconds}");
sw.Restart();
var diffImage = ImageUtils.GetImageDiff(frame1, frame2, false, out var hadChanges);
var diffImage = ImageUtils.GetImageDiff(frame1, frame2, false);
using (var ms = new MemoryStream())
{
diffImage.Save(ms, ImageFormat.Png);
diffImage.Value.Encode(ms, SKEncodedImageFormat.Png, 80);
Debug.WriteLine($"Diff image size: {ms.ToArray().Length}");
}
Debug.WriteLine($"Diff Image time: {sw.Elapsed.TotalMilliseconds}");
//sw.Restart();
//diffSize = 0;
//using (var tempImage = (Bitmap)frame1.Clone(new Rectangle(diff.X, diff.Y, diff.Width, diff.Height), PixelFormat.Format32bppArgb))
//{
// imageBytes = ImageUtils.EncodeWithSkia(tempImage, SkiaSharp.SKEncodedImageFormat.Webp, 60);
// diffSize = imageBytes.Length;
//}
//Debug.WriteLine($"WEBP diff size: {diffSize}");
//Debug.WriteLine($"WEBP diff time: {sw.Elapsed.TotalMilliseconds}");
//sw.Restart();
//diffImage = ImageUtils.GetImageDiff(frame1, frame2, false, out hadChanges);
//imageBytes = ImageUtils.EncodeWithSkia(diffImage, SkiaSharp.SKEncodedImageFormat.Webp, 60);
//Debug.WriteLine($"WEBP image size: {imageBytes.Length}");
//Debug.WriteLine($"WEBP Image time: {sw.Elapsed.TotalMilliseconds}");
Debug.WriteLine($"\n");
}
}
@ -131,23 +141,25 @@ namespace Remotely.Tests
_cursorWatcher = new Mock<ICursorIconWatcher>();
_sessionIndicator = new Mock<ISessionIndicator>();
_clipboard = new Mock<IClipboardService>();
_webrtcFactory = new Mock<IWebRtcSessionFactory>();
_casterSocket = new Mock<ICasterSocket>();
_audio = new Mock<IAudioCapturer>();
_shutdown = new Mock<IShutdownService>();
_webrtcFactory = new Mock<IWebRtcSessionFactory>();
_capturer = new ScreenCapturerWin();
_screenCaster = new ScreenCaster(_conductor, _cursorWatcher.Object, _sessionIndicator.Object, _shutdown.Object);
_viewer = new Viewer(_casterSocket.Object, _capturer, _clipboard.Object, _webrtcFactory.Object, _audio.Object);
_casterSocket
.Setup(x => x.SendDtoToViewer(It.IsAny<CaptureFrameDto>(), It.IsAny<string>()))
.Callback((CaptureFrameDto dto, string viewerId) => {
.Callback((CaptureFrameDto dto, string viewerId) =>
{
_bytesSent += dto.ImageBytes.Length;
});
_casterSocket
.Setup(x => x.SendDtoToViewer(It.Is<CaptureFrameDto>(x => x.EndOfFrame), It.IsAny<string>()))
.Callback(() => {
.Callback(() =>
{
Task.Run(() =>
{
Thread.Sleep(RoundtripLatency);
@ -171,16 +183,18 @@ namespace Remotely.Tests
ServiceContainer.Instance = serviceCollection.BuildServiceProvider();
}
private Bitmap GetFrame(string frameFileName)
private SKBitmap GetFrame(string frameFileName)
{
using var mrs = Assembly.GetExecutingAssembly().GetManifestResourceStream($"Remotely.Tests.Resources.{frameFileName}.jpg");
var resourceImage = (Bitmap)Bitmap.FromStream(mrs);
if (resourceImage.PixelFormat != PixelFormat.Format32bppArgb)
{
return resourceImage.Clone(new Rectangle(0, 0, resourceImage.Width, resourceImage.Height), PixelFormat.Format32bppArgb);
return resourceImage
.Clone(new Rectangle(0, 0, resourceImage.Width, resourceImage.Height), PixelFormat.Format32bppArgb)
.ToSKBitmap();
}
return resourceImage;
return resourceImage.ToSKBitmap();
}
private ImageCodecInfo GetEncoder(ImageFormat format)