Add some performance tests. Rotate DirectX captures if needed.

Desktop.Core/Interfaces/IScreenCapturer.cs

@@ -19,7 +19,7 @@ namespace Remotely.Desktop.Core.Interfaces
 
         Result<SKBitmap> GetImageDiff();
 
-        SKBitmap GetNextFrame();
+        Result<SKBitmap> GetNextFrame();
 
         int GetScreenCount();
 

Desktop.Core/Services/ScreenCaster.cs

@@ -92,13 +92,13 @@ namespace Remotely.Desktop.Core.Services
                 };
 
                 // This gets disposed internally in the Capturer on the next call.
-                var initialFrame = viewer.Capturer.GetNextFrame();
+                var result = viewer.Capturer.GetNextFrame();
 
-                if (initialFrame != null)
+                if (result.IsSuccess && result.Value is not null)
                 {
                     await viewer.SendScreenCapture(new CaptureFrame()
                     {
-                        EncodedImageBytes = ImageUtils.EncodeBitmap(initialFrame, SKEncodedImageFormat.Jpeg, viewer.ImageQuality),
+                        EncodedImageBytes = ImageUtils.EncodeBitmap(result.Value, SKEncodedImageFormat.Jpeg, viewer.ImageQuality),
                         Left = screenBounds.Left,
                         Top = screenBounds.Top,
                         Width = screenBounds.Width,
@@ -142,7 +142,12 @@ namespace Remotely.Desktop.Core.Services
 
                         viewer.ApplyAutoQuality();
 
-                        var currentFrame = viewer.Capturer.GetNextFrame();
+                        result = viewer.Capturer.GetNextFrame();
+
+                        if (!result.IsSuccess || result.Value is null)
+                        {
+                            continue;
+                        }
 
                         var diffArea = viewer.Capturer.GetFrameDiffArea();
 
@@ -153,7 +158,7 @@ namespace Remotely.Desktop.Core.Services
 
                         viewer.Capturer.CaptureFullscreen = false;
 
-                        using var croppedFrame = ImageUtils.CropBitmap(currentFrame, diffArea);
+                        using var croppedFrame = ImageUtils.CropBitmap(result.Value, diffArea);
 
                         var encodedImageBytes = ImageUtils.EncodeBitmap(croppedFrame, SKEncodedImageFormat.Jpeg, viewer.ImageQuality);
 

Desktop.Core/Services/WebRtcSession.cs

@@ -176,7 +176,15 @@ namespace Remotely.Desktop.Core.Services
                     return;
                 }
 
-                using var currentFrame = Viewer.Capturer.GetNextFrame();
+                var result = Viewer.Capturer.GetNextFrame();
+
+                if (!result.IsSuccess || result.Value is null)
+                {
+                    return;
+                }
+
+                using var currentFrame = result.Value;
+
                 if (currentFrame == null)
                 {
                     return;

Desktop.Win/Models/DirectXOutput.cs

@@ -2,6 +2,7 @@
 using SharpDX.Direct3D11;
 using SharpDX.DXGI;
 using System;
+using System.Drawing;
 
 namespace Remotely.Desktop.Win.Models
 {
@@ -18,17 +19,18 @@ namespace Remotely.Desktop.Win.Models
             OutputDuplication = outputDuplication;
             Texture2D = texture2D;
             Rotation = rotation;
+            Bounds = new Rectangle(0, 0, texture2D.Description.Width, texture2D.Description.Height);
         }
 
         public Adapter1 Adapter { get; }
+        public Rectangle Bounds { get; set; }
         public SharpDX.Direct3D11.Device Device { get; }
         public OutputDuplication OutputDuplication { get; }
         public DisplayModeRotation Rotation { get; }
         public Texture2D Texture2D { get; }
-
         public void Dispose()
         {
-            OutputDuplication.ReleaseFrame();
+            OutputDuplication?.ReleaseFrame();
             Disposer.TryDisposeAll(OutputDuplication, Texture2D, Adapter, Device);
             GC.SuppressFinalize(this);
         }

Desktop.Win/Properties/AssemblyInfo.cs

@@ -0,0 +1,3 @@
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("Remotely.Desktop.Win.Tests")]

Desktop.Win/Services/ScreenCapturerWin.cs

@@ -91,7 +91,7 @@ namespace Remotely.Desktop.Win.Services
             return ImageUtils.GetImageDiff(_currentFrame, _previousFrame);
         }
 
-        public SKBitmap GetNextFrame()
+        public Result<SKBitmap> GetNextFrame()
         {
             lock (_screenBoundsLock)
             {
@@ -107,35 +107,29 @@ namespace Remotely.Desktop.Win.Services
 
                     SwapFrames();
 
-                    // Sometimes DX will result in a timeout, even when there are changes
-                    // on the screen.  I've observed this when a laptop lid is closed, or
-                    // on some machines that aren't connected to a monitor.  This will
-                    // have it fall back to BitBlt in those cases.
-                    // TODO: Make DX capture work with changed screen orientation.
-                    if (_directxScreens.TryGetValue(SelectedScreen, out var dxDisplay) &&
-                        dxDisplay.Rotation == DisplayModeRotation.Identity)
-                    {
-                        var result = GetDirectXFrame();
+                    var result = GetDirectXFrame();
 
-                        if (result.IsSuccess && !IsEmpty(result.Value))
+                    if (!result.IsSuccess || result.Value is null || IsEmpty(result.Value))
+                    {
+                        result = GetBitBltFrame();
+                        if (!result.IsSuccess || result.Value is null)
                         {
-                            _currentFrame = result.Value;
-                            return _currentFrame;
+                            var ex = result.Exception ?? new("Unknown error.");
+                            Logger.Write(ex);
+                            return Result.Fail<SKBitmap>(ex);
                         }
                     }
 
-                    _currentFrame = GetBitBltFrame();
-                    return _currentFrame;
+                    _currentFrame = result.Value;
+                    return result;
                 }
                 catch (Exception e)
                 {
                     Logger.Write(e);
                     NeedsInit = true;
+                    return Result.Fail<SKBitmap>(e);
                 }
-
-                return null;
             }
-
         }
 
         public int GetScreenCount()
@@ -191,41 +185,27 @@ namespace Remotely.Desktop.Win.Services
             }
         }
 
-        private void ClearDirectXOutputs()
-        {
-            foreach (var screen in _directxScreens.Values)
-            {
-                try
-                {
-                    screen.Dispose();
-                }
-                catch { }
-            }
-            _directxScreens.Clear();
-        }
-
-        private SKBitmap GetBitBltFrame()
+        internal Result<SKBitmap> GetBitBltFrame()
         {
             try
             {
-                using var currentFrame = new Bitmap(CurrentScreenBounds.Width, CurrentScreenBounds.Height, PixelFormat.Format32bppArgb);
-                using (var graphic = Graphics.FromImage(currentFrame))
+                using var bitmap = new Bitmap(CurrentScreenBounds.Width, CurrentScreenBounds.Height, PixelFormat.Format32bppArgb);
+                using (var graphic = Graphics.FromImage(bitmap))
                 {
                     graphic.CopyFromScreen(CurrentScreenBounds.Left, CurrentScreenBounds.Top, 0, 0, new Size(CurrentScreenBounds.Width, CurrentScreenBounds.Height));
                 }
-                return currentFrame.ToSKBitmap();
+                return Result.Ok(bitmap.ToSKBitmap());
             }
             catch (Exception ex)
             {
                 Logger.Write(ex);
                 Logger.Write("Capturer error in BitBltCapture.");
                 NeedsInit = true;
+                return Result.Fail<SKBitmap>("Error while capturing BitBlt frame.");
             }
-
-            return null;
         }
 
-        private Result<SKBitmap> GetDirectXFrame()
+        internal Result<SKBitmap> GetDirectXFrame()
         {
             if (!_directxScreens.TryGetValue(SelectedScreen, out var dxOutput))
             {
@@ -237,9 +217,9 @@ namespace Remotely.Desktop.Win.Services
                 var outputDuplication = dxOutput.OutputDuplication;
                 var device = dxOutput.Device;
                 var texture2D = dxOutput.Texture2D;
-                var bounds = new Rectangle(0, 0, texture2D.Description.Width, texture2D.Description.Height);
+                var bounds = dxOutput.Bounds;
 
-                var result = outputDuplication.TryAcquireNextFrame(100, out var duplicateFrameInfo, out var screenResource);
+                var result = outputDuplication.TryAcquireNextFrame(50, out var duplicateFrameInfo, out var screenResource);
 
                 if (!result.Success)
                 {
@@ -265,26 +245,36 @@ namespace Remotely.Desktop.Win.Services
                 var bitmapDataPointer = bitmapData.Scan0;
                 for (var y = 0; y < bounds.Height; y++)
                 {
-                    SharpDX.Utilities.CopyMemory(bitmapDataPointer, dataBoxPointer, bounds.Width * 4);
+                    Utilities.CopyMemory(bitmapDataPointer, dataBoxPointer, bounds.Width * 4);
                     dataBoxPointer = IntPtr.Add(dataBoxPointer, dataBox.RowPitch);
                     bitmapDataPointer = IntPtr.Add(bitmapDataPointer, bitmapData.Stride);
                 }
                 bitmap.UnlockBits(bitmapData);
                 device.ImmediateContext.UnmapSubresource(texture2D, 0);
                 screenResource?.Dispose();
-                return Result.Ok(bitmap.ToSKBitmap());
-            }
-            catch (SharpDXException e)
-            {
-                if (e.ResultCode.Code == SharpDX.DXGI.ResultCode.WaitTimeout.Result.Code)
+
+                switch (dxOutput.Rotation)
                 {
-                    return Result.Fail<SKBitmap>("DirectX timed out while waiting for frame.");
+                    case DisplayModeRotation.Unspecified:
+                    case DisplayModeRotation.Identity:
+                        break;
+                    case DisplayModeRotation.Rotate90:
+                        bitmap.RotateFlip(RotateFlipType.Rotate270FlipNone);
+                        break;
+                    case DisplayModeRotation.Rotate180:
+                        bitmap.RotateFlip(RotateFlipType.Rotate180FlipNone);
+                        break;
+                    case DisplayModeRotation.Rotate270:
+                        bitmap.RotateFlip(RotateFlipType.Rotate90FlipNone);
+                        break;
+                    default:
+                        break;
                 }
-                Logger.Write(e);
+                return Result.Ok(bitmap.ToSKBitmap());
             }
             catch (Exception ex)
             {
-                Logger.Write(ex, "Error while grabbing with DirectX.");
+                Logger.Write(ex, "Error while getting DirectX frame.");
             }
             finally
             {
@@ -295,9 +285,21 @@ namespace Remotely.Desktop.Win.Services
                 catch { }
             }
 
-            return Result.Fail<SKBitmap>("Failed to get DirectX grab.");
+            return Result.Fail<SKBitmap>("Failed to get DirectX frame.");
         }
 
+        private void ClearDirectXOutputs()
+        {
+            foreach (var screen in _directxScreens.Values)
+            {
+                try
+                {
+                    screen.Dispose();
+                }
+                catch { }
+            }
+            _directxScreens.Clear();
+        }
         private void InitBitBlt()
         {
             _bitBltScreens.Clear();

Desktop.XPlat/Services/ScreenCapturerLinux.cs

@@ -53,7 +53,7 @@ namespace Remotely.Desktop.XPlat.Services
             return ImageUtils.GetImageDiff(_currentFrame, _previousFrame);
         }
 
-        public SKBitmap GetNextFrame()
+        public Result<SKBitmap> GetNextFrame()
         {
             lock (_screenBoundsLock)
             {
@@ -73,13 +73,13 @@ namespace Remotely.Desktop.XPlat.Services
                     }
 
                     _currentFrame = GetX11Capture();
-                    return _currentFrame;
+                    return Result.Ok(_currentFrame);
                 }
                 catch (Exception ex)
                 {
                     Logger.Write(ex);
                     Init();
-                    return null;
+                    return Result.Fail<SKBitmap>(ex);
                 }
             }
         }

Tests/Desktop.Win.Tests/Desktop.Win.Tests.csproj

@@ -13,13 +13,13 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <None Remove="Resources\Frame1.jpg" />
-    <None Remove="Resources\Frame2.jpg" />
+    <None Remove="Resources\Image1.jpg" />
+    <None Remove="Resources\Image2.jpg" />
   </ItemGroup>
 
   <ItemGroup>
-    <EmbeddedResource Include="Resources\Frame1.jpg" />
-    <EmbeddedResource Include="Resources\Frame2.jpg" />
+    <EmbeddedResource Include="Resources\Image1.jpg" />
+    <EmbeddedResource Include="Resources\Image2.jpg" />
   </ItemGroup>
 
   <ItemGroup>

Tests/Desktop.Win.Tests/ManualTests.cs

@@ -76,9 +76,8 @@ namespace Remotely.Tests
         {
             for (var i = 0; i < 2; i++)
             {
-                using var frame1 = GetFrame("Frame1");
-                using var frame2 = GetFrame("Frame2");
-                var jpegEncoder = GetEncoder(ImageFormat.Jpeg);
+                using var frame1 = GetImage("Image1");
+                using var frame2 = GetImage("Image1");
                 byte[] imageBytes;
 
                 var sw = Stopwatch.StartNew();
@@ -110,31 +109,191 @@ namespace Remotely.Tests
                 Debug.WriteLine($"Diff Image time: {sw.Elapsed.TotalMilliseconds}");
 
 
-                //sw.Restart();
-
-                //diffSize = 0;
-
-                //using (var tempImage = (Bitmap)frame1.Clone(new Rectangle(diff.X, diff.Y, diff.Width, diff.Height), PixelFormat.Format32bppArgb))
-                //{
-                //    imageBytes = ImageUtils.EncodeWithSkia(tempImage, SkiaSharp.SKEncodedImageFormat.Webp, 60);
-                //    diffSize = imageBytes.Length;
-                //}
-                //Debug.WriteLine($"WEBP diff size: {diffSize}");
-                //Debug.WriteLine($"WEBP diff time: {sw.Elapsed.TotalMilliseconds}");
-
-
-                //sw.Restart();
-                //diffImage = ImageUtils.GetImageDiff(frame1, frame2, false, out hadChanges);
-
-                //imageBytes = ImageUtils.EncodeWithSkia(diffImage, SkiaSharp.SKEncodedImageFormat.Webp, 60);
-                //Debug.WriteLine($"WEBP image size: {imageBytes.Length}");
-                //Debug.WriteLine($"WEBP Image time: {sw.Elapsed.TotalMilliseconds}");
-
 
                 Debug.WriteLine($"\n");
             }
         }
 
+
+        [TestMethod]
+        public void CaptureAndEncodeSpeedTest()
+        {
+            var iterations = 30;
+            var quality = 80;
+            var sw = Stopwatch.StartNew();
+
+            SKBitmap currentFrame = new();
+            SKBitmap previousFrame = new();
+
+            for (var i = 0; i < iterations; i++)
+            {
+                previousFrame?.Dispose();
+                previousFrame = currentFrame.Copy();
+                currentFrame.Dispose();
+                
+                currentFrame = _capturer.GetNextFrame().Value;
+                var diffArea = ImageUtils.GetDiffArea(currentFrame, previousFrame);
+                using var cropped = ImageUtils.CropBitmap(currentFrame, diffArea);
+                using var skData = cropped.Encode(SKEncodedImageFormat.Webp, quality);
+            }
+            sw.Stop();
+            Console.WriteLine($"GetNextFrame & WEBP: {GetAverage(sw, iterations)}ms per iteration");
+
+            sw.Restart();
+            for (var i = 0; i < iterations; i++)
+            {
+                previousFrame.Dispose();
+                previousFrame = currentFrame.Copy();
+                currentFrame.Dispose();
+
+                currentFrame = _capturer.GetNextFrame().Value;
+                var diffArea = ImageUtils.GetDiffArea(currentFrame, previousFrame);
+                using var cropped = ImageUtils.CropBitmap(currentFrame, diffArea);
+                using var skData = cropped.Encode(SKEncodedImageFormat.Jpeg, quality);
+            }
+            sw.Stop();
+            Console.WriteLine($"GetNextFrame & JPEG: {GetAverage(sw, iterations)}ms per iteration");
+        }
+
+        [TestMethod]
+        public void CaptureSpeedTest()
+        {
+            var iterations = 30;
+            var sw = Stopwatch.StartNew();
+
+            for (var i = 0; i < iterations; i++)
+            {
+                using var bitmap = _capturer.GetNextFrame().Value;
+            }
+
+            sw.Stop();
+
+            Console.WriteLine($"GetNextFrame: {GetAverage(sw, iterations)}ms per capture");
+
+
+
+            sw.Restart();
+
+            for (var i = 0; i < iterations; i++)
+            {
+                using var bitmap = _capturer.GetDirectXFrame().Value;
+            }
+
+            sw.Stop();
+
+            Console.WriteLine($"DirectX: {GetAverage(sw, iterations)}ms per capture");
+
+
+
+            sw.Restart();
+
+            for (var i = 0; i < iterations; i++)
+            {
+                using var bitmap = _capturer.GetBitBltFrame().Value;
+            }
+
+            sw.Stop();
+
+            Console.WriteLine($"BitBlt: {GetAverage(sw, iterations)}ms per capture");
+
+        }
+
+        [TestMethod]
+        public void DiffSpeedTests()
+        {
+            using var bitmap1 = GetImage("Image1");
+            using var bitmap2 = GetImage("Image2");
+            var iterations = 60;
+
+
+            var sw = Stopwatch.StartNew();
+            for (var i = 0; i < iterations; i++)
+            {
+                _ = ImageUtils.GetDiffArea(bitmap1, bitmap2);
+            }
+            sw.Stop();
+            Console.WriteLine($"Diff Area: {GetAverage(sw, iterations)}ms per call");
+
+
+            sw.Restart();
+            for (var i = 0; i < iterations; i++)
+            {
+                using var imageDiff = ImageUtils.GetImageDiff(bitmap1, bitmap2).Value;
+            }
+            sw.Stop();
+            Console.WriteLine($"Image Diff: {GetAverage(sw, iterations)}ms per call");
+        }
+
+        [TestMethod]
+        public void EncodeSpeedTest()
+        {
+            using var skBitmap = GetImage("Image1");
+            var quality = 75;
+            var iterations = 30;
+
+            {
+                using var skData = skBitmap.Encode(SKEncodedImageFormat.Jpeg, quality);
+                Console.WriteLine($"JPEG size: {skData.Size:N0}");
+            }
+
+            var sw = Stopwatch.StartNew();
+            for (var i = 0; i < iterations; i++)
+            {
+                using var skData = skBitmap.Encode(SKEncodedImageFormat.Jpeg, quality);
+            }
+            sw.Stop();
+            Console.WriteLine($"JPEG: {GetAverage(sw, iterations)}ms per encode");
+
+
+            {
+                using var skData = skBitmap.Encode(SKEncodedImageFormat.Png, quality);
+                Console.WriteLine($"PNG size: {skData.Size:N0}");
+            }
+            sw.Restart();
+            for (var i = 0; i < iterations; i++)
+            {
+                using var skData = skBitmap.Encode(SKEncodedImageFormat.Png, quality);
+            }
+            sw.Stop();
+            Console.WriteLine($"PNG: {GetAverage(sw, iterations)}ms per encode");
+
+
+            {
+                using var skData = skBitmap.Encode(SKEncodedImageFormat.Webp, quality);
+                Console.WriteLine($"WEBP size: {skData.Size:N0}");
+            }
+            sw.Restart();
+            for (var i = 0; i < iterations; i++)
+            {
+                using var skData = skBitmap.Encode(SKEncodedImageFormat.Webp, quality);
+            }
+            sw.Stop();
+            Console.WriteLine($"WEBP: {GetAverage(sw, iterations)}ms per encode");
+        }
+
+        [TestMethod]
+        public void GetDiffAreaTest()
+        {
+            using var bitmap1 = GetImage("Image1");
+            using var bitmap2 = GetImage("Image2");
+
+            var diffArea = ImageUtils.GetDiffArea(bitmap1, bitmap2);
+            using var cropped = ImageUtils.CropBitmap(bitmap2, diffArea);
+
+            SaveFile(cropped, "Test.webp");
+        }
+
+        [TestMethod]
+        public void GetImageDiffTest()
+        {
+            using var bitmap1 = GetImage("Image1");
+            using var bitmap2 = GetImage("Image2");
+
+            var diff = ImageUtils.GetImageDiff(bitmap1, bitmap2);
+
+            SaveFile(diff.Value, "Test.webp");
+        }
+
         [TestInitialize]
         public void Init()
         {
@@ -184,9 +343,15 @@ namespace Remotely.Tests
             ServiceContainer.Instance = serviceCollection.BuildServiceProvider();
         }
 
-        private SKBitmap GetFrame(string frameFileName)
+
+        private static double GetAverage(Stopwatch sw, int iterations)
         {
-            using var mrs = Assembly.GetExecutingAssembly().GetManifestResourceStream($"Remotely.Desktop.Win.Tests.Resources.{frameFileName}.jpg");
+            return Math.Round(sw.Elapsed.TotalMilliseconds / iterations, 2);
+        }
+
+        private SKBitmap GetImage(string imageFileName)
+        {
+            using var mrs = Assembly.GetExecutingAssembly().GetManifestResourceStream($"Remotely.Desktop.Win.Tests.Resources.{imageFileName}.jpg");
             var resourceImage = (Bitmap)Bitmap.FromStream(mrs);
 
             if (resourceImage.PixelFormat != PixelFormat.Format32bppArgb)
@@ -198,11 +363,15 @@ namespace Remotely.Tests
             return resourceImage.ToSKBitmap();
         }
 
-        private ImageCodecInfo GetEncoder(ImageFormat format)
+        private static void SaveFile(
+          SKBitmap bitmap,
+          string fileName,
+          SKEncodedImageFormat format = SKEncodedImageFormat.Webp,
+          int quality = 80)
         {
-            var codecs = ImageCodecInfo.GetImageEncoders();
-
-            return codecs.FirstOrDefault(x => x.FormatID == format.Guid);
+            var savePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), fileName);
+            using var fs = new FileStream(savePath, FileMode.Create);
+            bitmap.Encode(fs, format, quality);
         }
     }
 }