mirrors/Remotely
3
0
Fork 0
mirror of https://github.com/immense/Remotely.git synced 2025-10-26 11:27:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Change encoding and image diffing.

Browse Source
This commit is contained in:
Jared Goodwin 2019-03-09 21:46:46 -08:00
parent 4a4e6c40a4
commit 9ef92d67b8
10 changed files with 206 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
Remotely_ScreenCast/Capture/ImageUtils.cs
View File

@ -14,16 +14,107 @@ namespace Remotely_ScreenCast.Capture
{
public class ImageUtils
{
private static EncoderParameters EncoderParams { get; } = new EncoderParameters()
{
Param = new EncoderParameter[]
{
new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, 0L),
new EncoderParameter(System.Drawing.Imaging.Encoder.ColorDepth, 8L)
}
};
private static ImageCodecInfo CodecInfo { get; } = ImageCodecInfo.GetImageEncoders().FirstOrDefault(x => x.FormatID == ImageFormat.Jpeg.Guid);
private static ImageCodecInfo CodecInfo { get; } = ImageCodecInfo.GetImageEncoders().FirstOrDefault(x => x.FormatID == ImageFormat.Png.Guid);
public static byte[] EncodeBitmap(Bitmap bitmap, EncoderParameters encoderParams)
{
using (var ms = new MemoryStream())
{
bitmap.Save(ms, CodecInfo, encoderParams);
return ms.ToArray();
}
}
public static Rectangle GetDiffArea(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen)
{
if (captureFullscreen)
{
return new Rectangle(new Point(0, 0), currentFrame.Size);
}
if (currentFrame.Height != previousFrame.Height || currentFrame.Width != previousFrame.Width)
{
throw new Exception("Bitmaps are not of equal dimensions.");
}
if (!Bitmap.IsAlphaPixelFormat(currentFrame.PixelFormat) || !Bitmap.IsAlphaPixelFormat(previousFrame.PixelFormat) ||
!Bitmap.IsCanonicalPixelFormat(currentFrame.PixelFormat) || !Bitmap.IsCanonicalPixelFormat(previousFrame.PixelFormat))
{
throw new Exception("Bitmaps must be 32 bits per pixel and contain alpha channel.");
}
var width = currentFrame.Width;
var height = currentFrame.Height;
int left = int.MaxValue;
int top = int.MaxValue;
int right = int.MinValue;
int bottom = int.MinValue;
var bd1 = previousFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, currentFrame.PixelFormat);
var bd2 = currentFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, previousFrame.PixelFormat);
// Get the address of the first line.
IntPtr ptr1 = bd1.Scan0;
IntPtr ptr2 = bd2.Scan0;
// Declare an array to hold the bytes of the bitmap.
int arraySize = Math.Abs(bd1.Stride) * currentFrame.Height;
var rgbValues1 = new byte[arraySize];
var rgbValues2 = new byte[arraySize];
// Copy the RGBA values into the array.
Marshal.Copy(ptr1, rgbValues1, 0, arraySize);
Marshal.Copy(ptr2, rgbValues2, 0, arraySize);
// Check RGBA value for each pixel.
for (int counter = 0; counter < rgbValues1.Length - 4; counter += 4)
{
if (rgbValues1[counter] != rgbValues2[counter] ||
rgbValues1[counter + 1] != rgbValues2[counter + 1] ||
rgbValues1[counter + 2] != rgbValues2[counter + 2] ||
rgbValues1[counter + 3] != rgbValues2[counter + 3])
{
// Change was found.
var pixel = counter / 4;
var row = (int)Math.Floor((double)pixel / bd1.Width);
var column = pixel % bd1.Width;
if (row < top)
{
top = row;
}
if (row > bottom)
{
bottom = row;
}
if (column < left)
{
left = column;
}
if (column > right)
{
right = column;
}
}
}
if (left < right && top < bottom)
{
// Bounding box is valid.
left = Math.Max(left - 20, 0);
top = Math.Max(top - 20, 0);
right = Math.Min(right + 20, width);
bottom = Math.Min(bottom + 20, height);
currentFrame.UnlockBits(bd1);
previousFrame.UnlockBits(bd2);
return new Rectangle(left, top, right - left, bottom - top);
}
else
{
currentFrame.UnlockBits(bd1);
previousFrame.UnlockBits(bd2);
return Rectangle.Empty;
}
}
public static Bitmap GetImageDiff(Bitmap currentFrame, Bitmap previousFrame, bool captureFullscreen)
{
@ -48,22 +139,22 @@ namespace Remotely_ScreenCast.Capture
var bd1 = previousFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, currentFrame.PixelFormat);
var bd2 = currentFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.ReadOnly, previousFrame.PixelFormat);
var bd3 = mergedFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.WriteOnly, currentFrame.PixelFormat);
var bd3 = mergedFrame.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.WriteOnly, currentFrame.PixelFormat);
// Get the address of the first line.
IntPtr ptr1 = bd1.Scan0;
// Get the address of the first line.
IntPtr ptr1 = bd1.Scan0;
IntPtr ptr2 = bd2.Scan0;
IntPtr ptr3 = bd3.Scan0;
IntPtr ptr3 = bd3.Scan0;
// Declare an array to hold the bytes of the bitmap.
int arraySize = Math.Abs(bd1.Stride) * currentFrame.Height;
// Declare an array to hold the bytes of the bitmap.
int arraySize = Math.Abs(bd1.Stride) * currentFrame.Height;
var rgbValues1 = new byte[arraySize];
var rgbValues2 = new byte[arraySize];
var rgbValues3 = new byte[arraySize];
var rgbValues3 = new byte[arraySize];
// Copy the RGBA values into the array.
Marshal.Copy(ptr1, rgbValues1, 0, arraySize);
// Copy the RGBA values into the array.
Marshal.Copy(ptr1, rgbValues1, 0, arraySize);
Marshal.Copy(ptr2, rgbValues2, 0, arraySize);
// Check RGBA value for each pixel.
@ -74,31 +165,22 @@ namespace Remotely_ScreenCast.Capture
rgbValues1[counter + 2] != rgbValues2[counter + 2] ||
rgbValues1[counter + 3] != rgbValues2[counter + 3])
{
// Change was found.
rgbValues3[counter] = rgbValues2[counter];
rgbValues3[counter + 1] = rgbValues2[counter + 1];
rgbValues3[counter + 2] = rgbValues2[counter + 2];
rgbValues3[counter + 3] = rgbValues2[counter + 3];
}
// Change was found.
rgbValues3[counter] = rgbValues2[counter];
rgbValues3[counter + 1] = rgbValues2[counter + 1];
rgbValues3[counter + 2] = rgbValues2[counter + 2];
rgbValues3[counter + 3] = rgbValues2[counter + 3];
}
}
// Copy merged frame to bitmap.
Marshal.Copy(rgbValues3, 0, ptr3, rgbValues3.Length);
// Copy merged frame to bitmap.
Marshal.Copy(rgbValues3, 0, ptr3, rgbValues3.Length);
previousFrame.UnlockBits(bd1);
previousFrame.UnlockBits(bd1);
currentFrame.UnlockBits(bd2);
mergedFrame.UnlockBits(bd3);
mergedFrame.UnlockBits(bd3);
return mergedFrame;
}
public static byte[] EncodeBitmap(Bitmap bitmap)
{
using (var ms = new MemoryStream())
{
bitmap.Save(ms, CodecInfo, EncoderParams);
return ms.ToArray();
}
}
}
}

50
Remotely_ScreenCast/Capture/ScreenCaster.cs
View File

@ -1,9 +1,11 @@
using Microsoft.AspNetCore.SignalR.Client;
using Remotely_ScreenCast.Models;
using Remotely_ScreenCast.Sockets;
using Remotely_ScreenCast.Utilities;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
@ -20,6 +22,8 @@ namespace Remotely_ScreenCast.Capture
{
ICapturer capturer;
CaptureMode captureMode;
Viewer viewer;
var success = false;
try
{
@ -48,16 +52,16 @@ namespace Remotely_ScreenCast.Capture
Logger.Write($"Starting screen cast. Requester: {requesterName}. Viewer ID: {viewerID}. Capture Mode: {captureMode.ToString()}. App Mode: {Program.Mode} Desktop: {Program.CurrentDesktopName}");
var viewer = new Models.Viewer()
viewer = new Viewer()
{
Capturer = capturer,
DisconnectRequested = false,
Name = requesterName,
ViewerConnectionID = viewerID,
HasControl = Program.Mode == Enums.AppMode.Unattended
HasControl = Program.Mode == Enums.AppMode.Unattended,
ImageQuality = 1
};
var success = false;
while (!success)
{
success = Program.Viewers.TryAdd(viewerID, viewer);
@ -97,35 +101,47 @@ namespace Remotely_ScreenCast.Capture
while (viewer.PendingFrames > 10)
{
Logger.Write("Waiting on pending frames.");
await Task.Delay(1);
}
capturer.Capture();
var newImage = ImageUtils.GetImageDiff(capturer.CurrentFrame, capturer.PreviousFrame, capturer.CaptureFullscreen);
var diffArea = ImageUtils.GetDiffArea(capturer.CurrentFrame, capturer.PreviousFrame, capturer.CaptureFullscreen);
if (viewer.PendingFrames > 5)
{
var reductionRatio = (double)5 / viewer.PendingFrames;
Logger.Write($"Reducing image quality to {reductionRatio}.");
newImage = new Bitmap(
capturer.CurrentFrame,
(int)(capturer.CurrentScreenBounds.Width * reductionRatio),
(int)(capturer.CurrentScreenBounds.Height * reductionRatio));
}
var newImage = capturer.CurrentFrame.Clone(diffArea, System.Drawing.Imaging.PixelFormat.Format32bppArgb);
if (capturer.CaptureFullscreen)
{
capturer.CaptureFullscreen = false;
}
var img = ImageUtils.EncodeBitmap(newImage);
long newQuality;
if (viewer.PendingFrames < 5)
{
newQuality = (long)Math.Min(1, viewer.ImageQuality + .1);
}
else
{
newQuality = (long)Math.Max(.1, viewer.ImageQuality - .1);
}
Logger.Write($"New quality: {newQuality}");
if (newQuality != viewer.ImageQuality)
{
viewer.ImageQuality = newQuality;
viewer.FullScreenRefreshNeeded = true;
}
else
{
capturer.CaptureFullscreen = true;
viewer.FullScreenRefreshNeeded = false;
}
var img = ImageUtils.EncodeBitmap(newImage, viewer.EncoderParams);
if (img?.Length > 0)
{
await outgoingMessages.SendScreenCapture(img, viewerID, DateTime.UtcNow);
await outgoingMessages.SendScreenCapture(img, viewerID, diffArea.Left, diffArea.Top, diffArea.Width, diffArea.Height, DateTime.UtcNow);
viewer.PendingFrames++;
}
}

37
Remotely_ScreenCast/Models/Viewer.cs
View File

@ -2,6 +2,7 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
@ -15,7 +16,41 @@ namespace Remotely_ScreenCast.Models
public ICapturer Capturer { get; set; }
public bool DisconnectRequested { get; set; }
public bool HasControl { get; set; }
public double Latency { get; set; }
public double Latency { get; set; } = 1;
public int PendingFrames { get; set; }
private long imageQuality = 1;
public long ImageQuality
{
get
{
return imageQuality;
}
set
{
if (imageQuality > 100 || imageQuality < 0)
{
return;
}
imageQuality = value;
EncoderParams = new EncoderParameters()
{
Param = new EncoderParameter[]
{
new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, value)
}
};
}
}
public bool FullScreenRefreshNeeded { get; internal set; }
public EncoderParameters EncoderParams { get; private set; } = new EncoderParameters()
{
Param = new EncoderParameter[]
{
new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, 75L)
}
};
}
}

2
Remotely_ScreenCast/Sockets/MessageHandlers.cs
View File

@ -169,7 +169,7 @@ namespace Remotely_ScreenCast.Sockets
}
await hubConnection.InvokeAsync("ViewerDisconnected", viewerID);
});
hubConnection.On("LatencyUpdate", (double latency, double payloadSize, string viewerID) =>
hubConnection.On("LatencyUpdate", (double latency, string viewerID) =>
{
if (Program.Viewers.TryGetValue(viewerID, out var viewer))
{

4
Remotely_ScreenCast/Sockets/OutgoingMessages.cs
View File

@ -22,9 +22,9 @@ namespace Remotely_ScreenCast.Sockets
await Connection.SendAsync("SendScreenSize", width, height, viewerID);
}
public async Task SendScreenCapture(byte[] captureBytes, string viewerID, DateTime captureTime)
public async Task SendScreenCapture(byte[] captureBytes, string viewerID, int left, int top, int width, int height, DateTime captureTime)
{
await Connection.SendAsync("SendScreenCapture", captureBytes, viewerID, captureTime);
await Connection.SendAsync("SendScreenCapture", captureBytes, viewerID, left, top, width, height, captureTime);
}
internal async Task SendScreenCount(int primaryScreenIndex, int screenCount, string viewerID)

4
Remotely_Server/Services/RCBrowserSocketHub.cs
View File

@ -159,9 +159,9 @@ namespace Remotely_Server.Services
await RCDeviceHub.Clients.Client(screenCasterID).SendAsync("GetScreenCast", Context.ConnectionId, requesterName);
}
public async Task SendLatencyUpdate(double latency, double payloadSize)
public async Task SendLatencyUpdate(double latency)
{
await RCDeviceHub.Clients.Client(ScreenCasterID).SendAsync("LatencyUpdate", latency, payloadSize, Context.ConnectionId);
await RCDeviceHub.Clients.Client(ScreenCasterID).SendAsync("LatencyUpdate", latency, Context.ConnectionId);
}
public async Task SendSharedFileIDs(List<string> fileIDs)
{

4
Remotely_Server/Services/RCDeviceSocketHub.cs
View File

@ -106,9 +106,9 @@ namespace Remotely_Server.Services
await RCBrowserHub.Clients.Client(rcBrowserHubConnectionID).SendAsync("ScreenSize", width, height);
}
public Task SendScreenCapture(byte[] captureBytes, string rcBrowserHubConnectionID, DateTime captureTime)
public Task SendScreenCapture(byte[] captureBytes, string rcBrowserHubConnectionID, int left, int top, int width, int height, DateTime captureTime)
{
return RCBrowserHub.Clients.Client(rcBrowserHubConnectionID).SendAsync("ScreenCapture", captureBytes, captureTime);
return RCBrowserHub.Clients.Client(rcBrowserHubConnectionID).SendAsync("ScreenCapture", captureBytes, left, top, width, height, captureTime);
}
public async Task NotifyRequesterUnattendedReady(string browserHubConnectionID)

10
Remotely_Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js
View File

@ -31,8 +31,8 @@ export class RCBrowserSockets {
SendScreenCastRequestToDevice() {
return this.Connection.invoke("SendScreenCastRequestToDevice", RemoteControl.ClientID, RemoteControl.RequesterName, RemoteControl.Mode);
}
SendLatencyUpdate(latency, payloadSize) {
this.Connection.invoke("SendLatencyUpdate", latency, payloadSize);
SendLatencyUpdate(latency) {
this.Connection.invoke("SendLatencyUpdate", latency);
}
SendSelectScreen(index) {
return this.Connection.invoke("SelectScreen", index);
@ -104,13 +104,13 @@ export class RCBrowserSockets {
UI.ScreenViewer.height = height;
UI.Screen2DContext.clearRect(0, 0, width, height);
});
hubConnection.on("ScreenCapture", (buffer, captureTime) => {
hubConnection.on("ScreenCapture", (buffer, left, top, width, height, captureTime) => {
var latency = Date.now() - new Date(captureTime).getTime();
this.SendLatencyUpdate(latency, buffer.length);
this.SendLatencyUpdate(latency);
var url = window.URL.createObjectURL(new Blob([buffer]));
var img = document.createElement("img");
img.onload = () => {
UI.Screen2DContext.drawImage(img, 0, 0, UI.ScreenViewer.width, UI.ScreenViewer.height);
UI.Screen2DContext.drawImage(img, left, top, width, height);
window.URL.revokeObjectURL(url);
};
img.src = url;

2
Remotely_Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.js.map
View File

File diff suppressed because one or more lines are too long

10
Remotely_Server/wwwroot/scripts/RemoteControl/RCBrowserSockets.ts
View File

@ -38,8 +38,8 @@ export class RCBrowserSockets {
SendScreenCastRequestToDevice() {
return this.Connection.invoke("SendScreenCastRequestToDevice", RemoteControl.ClientID, RemoteControl.RequesterName, RemoteControl.Mode);
}
SendLatencyUpdate(latency: number, payloadSize: number) {
this.Connection.invoke("SendLatencyUpdate", latency, payloadSize);
SendLatencyUpdate(latency: number) {
this.Connection.invoke("SendLatencyUpdate", latency);
}
SendSelectScreen(index: number) {
return this.Connection.invoke("SelectScreen", index);
@ -112,14 +112,14 @@ export class RCBrowserSockets {
UI.ScreenViewer.height = height;
UI.Screen2DContext.clearRect(0, 0, width, height);
});
hubConnection.on("ScreenCapture", (buffer: Uint8Array, captureTime: Date) => {
hubConnection.on("ScreenCapture", (buffer: Uint8Array, left:number, top:number, width:number, height:number, captureTime: Date) => {
var latency = Date.now() - new Date(captureTime).getTime();
this.SendLatencyUpdate(latency, buffer.length);
this.SendLatencyUpdate(latency);
var url = window.URL.createObjectURL(new Blob([buffer]));
var img = document.createElement("img");
img.onload = () => {
UI.Screen2DContext.drawImage(img, 0, 0, UI.ScreenViewer.width, UI.ScreenViewer.height);
UI.Screen2DContext.drawImage(img, left, top, width, height);
window.URL.revokeObjectURL(url);
};
img.src = url;