public void ProcessFrame(ProcessVideoFrameContext context) { using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = output.CreateDrawingSession()) { TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan(); ds.Clear(Colors.Black); for (uint i = 0; i < numColumns; i++) { for (uint j = 0; j < numRows; j++) { crops[i, j].Source = input; float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f); float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j); Vector2 centerPoint = new Vector2((i + 0.5f) * pixelsPerTile, (j + 0.5f) * pixelsPerTile); transforms[i, j].TransformMatrix = Matrix3x2.CreateRotation(rotation, centerPoint) * Matrix3x2.CreateScale(scale, centerPoint); ds.DrawImage(transforms[i, j]); } } } }
public void ProcessFrameGPU(ProcessVideoFrameContext context) { var inputSurface = context.InputFrame.Direct3DSurface; var outputSurface = context.OutputFrame.Direct3DSurface; using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(m_canvasDevice, inputSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(m_canvasDevice, outputSurface)) using (var drawingSession = renderTarget.CreateDrawingSession()) { var normalize = new Matrix5x4() { M11 = 1f, M22 = 1f, M33 = 1f, M44 = 1f, M51 = -m_mean.X, M52 = -m_mean.Y, M53 = -m_mean.Z }; var normalize2 = new Matrix5x4() { M11 = 1 / m_std.X, M22 = 1 / m_std.Y, M33 = 1 / m_std.Z, M44 = 1.0f }; // https://microsoft.github.io/Win2D/html/T_Microsoft_Graphics_Canvas_Effects_ColorMatrixEffect.htm var PreprocessTransfrom = new ColorMatrixEffect() { ColorMatrix = normalize2, Source = new ColorMatrixEffect() { ColorMatrix = normalize, Source = inputBitmap } }; drawingSession.DrawImage(PreprocessTransfrom); } }
public void CompositeFrame(CompositeVideoFrameContext context) { IDirect3DSurface outputSurface = context.OutputFrame.Direct3DSurface; using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, outputSurface)) using (CanvasDrawingSession drawSession = renderTarget.CreateDrawingSession()) { foreach (var overlaySurface in context.SurfacesToOverlay) { var overlay = context.GetOverlayForSurface(overlaySurface); var width = (float)overlay.Position.Width; var height = (float)overlay.Position.Height; using (var overlayBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, overlaySurface)) using (var videoBrush = new CanvasImageBrush(_canvasDevice, overlayBitmap)) { var scale = width / overlay.Clip.GetVideoEncodingProperties().Width; videoBrush.Transform = Matrix3x2.CreateScale(scale) * Matrix3x2.CreateTranslation((float)overlay.Position.X, (float)overlay.Position.Y); drawSession.FillEllipse(new Vector2((float)overlay.Position.X + width / 2, (float)overlay.Position.Y + height / 2), width / 2, height / 2, videoBrush); } } drawSession.DrawText("Party\nTime!", new Vector2(_backgroundProperties.Width / 1.5f, 100), Windows.UI.Colors.CornflowerBlue, new CanvasTextFormat() { FontSize = (float)_backgroundProperties.Width / 13, FontWeight = new FontWeight() { Weight = 999 }, HorizontalAlignment = CanvasHorizontalAlignment.Center, VerticalAlignment = CanvasVerticalAlignment.Center }); } }
public void ProcessFrame(ProcessVideoFrameContext context) { var inputSurface = context.InputFrame.Direct3DSurface; var outputSurface = context.OutputFrame.Direct3DSurface; using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, inputSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, outputSurface)) using (var ds = renderTarget.CreateDrawingSession()) using (var brush = new CanvasImageBrush(canvasDevice, inputBitmap)) using (var textCommandList = new CanvasCommandList(canvasDevice)) { using (var clds = textCommandList.CreateDrawingSession()) { clds.DrawText( "Win2D\nMediaClip", (float)inputBitmap.Size.Width / 2, (float)inputBitmap.Size.Height / 2, brush, new CanvasTextFormat() { FontSize = (float)inputBitmap.Size.Width / 5, FontWeight = new FontWeight() { Weight = 999 }, HorizontalAlignment = CanvasHorizontalAlignment.Center, VerticalAlignment = CanvasVerticalAlignment.Center }); } var background = new GaussianBlurEffect() { BlurAmount = 10, BorderMode = EffectBorderMode.Hard, Source = new BrightnessEffect() { BlackPoint = new Vector2(0.5f, 0.7f), Source = new SaturationEffect() { Saturation = 0, Source = inputBitmap } } }; var shadow = new ShadowEffect() { Source = textCommandList, BlurAmount = 10 }; var composite = new CompositeEffect() { Sources = { background, shadow, textCommandList } }; ds.DrawImage(composite); } }
// ** Methods ** // // This is run for every video frame passed in the media pipeline (MediaPlayer, MediaCapture, etc) public void ProcessFrame(ProcessVideoFrameContext context) { videoFrameToProcess = VideoFrame.CreateWithDirect3D11Surface(context.InputFrame.Direct3DSurface); // ********** Draw Bounding Boxes with Win2D ********** // // Use Direct3DSurface if using GPU memory if (context.InputFrame.Direct3DSurface != null) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) { ds.DrawImage(inputBitmap); foreach (var box in filteredBoxes) { var x = (uint)Math.Max(box.X, 0); var y = (uint)Math.Max(box.Y, 0); var w = (uint)Math.Min(renderTarget.Bounds.Width - x, box.Width); var h = (uint)Math.Min(renderTarget.Bounds.Height - y, box.Height); // Draw the Text 10px above the top of the bounding box ds.DrawText(box.Label, x, y - 10, Colors.Yellow); ds.DrawRectangle(new Rect(x, y, w, h), new CanvasSolidColorBrush(canvasDevice, Colors.Yellow), 2f); } } return; } // Use SoftwareBitmap if using CPU memory if (context.InputFrame.SoftwareBitmap != null) { // InputFrame's pixels byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight]; context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer()); using (var inputBitmap = CanvasBitmap.CreateFromBytes(canvasDevice, inputFrameBytes, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat())) using (var renderTarget = new CanvasRenderTarget(canvasDevice, context.OutputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, (float)context.OutputFrame.SoftwareBitmap.DpiX, context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(), CanvasAlphaMode.Premultiplied)) using (var ds = renderTarget.CreateDrawingSession()) { ds.DrawImage(inputBitmap); foreach (var box in filteredBoxes) { var x = (uint)Math.Max(box.X, 0); var y = (uint)Math.Max(box.Y, 0); var w = (uint)Math.Min(context.OutputFrame.SoftwareBitmap.PixelWidth - x, box.Width); var h = (uint)Math.Min(context.OutputFrame.SoftwareBitmap.PixelHeight - y, box.Height); // Draw the Text 10px above the top of the bounding box ds.DrawText(box.Label, x, y - 10, Colors.Yellow); ds.DrawRectangle(new Rect(x, y, w, h), new CanvasSolidColorBrush(canvasDevice, Colors.Yellow), 2f); } } } }
public void ProcessFrame(ProcessVideoFrameContext context) { // If memory type is CPU, the frame is in InputFrame.SoftwareBitmap. // For GPU, the frame is in InputFrame.Direct3DSurface if (context.InputFrame.SoftwareBitmap == null) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) { var sepia = new SepiaEffect { Source = inputBitmap, Intensity = this.Intensity }; ds.DrawImage(sepia); } return; } if (context.InputFrame.Direct3DSurface == null) { // InputFrame's raw pixels byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight]; context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer()); using (var inputBitmap = CanvasBitmap.CreateFromBytes( _canvasDevice, inputFrameBytes, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat())) using (var renderTarget = new CanvasRenderTarget( _canvasDevice, context.OutputFrame.SoftwareBitmap.PixelWidth, context.OutputFrame.SoftwareBitmap.PixelHeight, (float)context.OutputFrame.SoftwareBitmap.DpiX, context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(), CanvasAlphaMode.Premultiplied)) { using (var ds = renderTarget.CreateDrawingSession()) { var sepia = new SepiaEffect { Source = inputBitmap, Intensity = this.Intensity }; ds.DrawImage(sepia); } } } }
public void ProcessFrame(ProcessVideoFrameContext context) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, context.InputFrame.Direct3DSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) { ds.Transform = MediaExtension.CreateTransform(inputBitmap); ds.DrawImage(inputBitmap); } }
public void ProcessFrame(ProcessVideoFrameContext context) { bool skipMaskPred = false; if (context.InputFrame.IsDiscontinuous) { streamStartDelta = TimeSpan.FromTicks(DateTime.Now.Ticks) - context.InputFrame.SystemRelativeTime.Value; } else { if ((TimeSpan.FromTicks(DateTime.Now.Ticks) - context.InputFrame.SystemRelativeTime.Value - streamStartDelta) > TimeSpan.FromMilliseconds(maxDelay)) { skipMaskPred = true; } } if (!skipMaskPred) { frameCount++; features["0"] = context.InputFrame; var resTask = _learningModel.EvaluateFeaturesAsync(features, string.Empty).AsTask(); var startTime = DateTime.Now.Ticks; resTask.Wait(); Debug.WriteLine("delta {0}", TimeSpan.FromTicks(DateTime.Now.Ticks - startTime)); } using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasBitmap inputMask = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, output.Direct3DSurface)) using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession()) { ds.Clear(Colors.Green); var addAlpha = new ColorMatrixEffect() { Source = inputMask, ColorMatrix = RToAlpha }; var resize = new ScaleEffect() { Source = addAlpha, Scale = new Vector2(((float)inputBitmap.SizeInPixels.Width / inputMask.SizeInPixels.Width), ((float)inputBitmap.SizeInPixels.Height / inputMask.SizeInPixels.Height)) }; var blend = new AlphaMaskEffect() { Source = inputBitmap, AlphaMask = resize }; ds.DrawImage(blend); ds.DrawText(String.Format("FPS: {0:f1}", currentFPS), fpsLabelOffset, fpsLabelColor); } }
//private ICanvasImage[] BackgroundImages { get; set; } = default(ICanvasImage[]); public void ProcessFrame(ProcessVideoFrameContext context) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, context.InputFrame.Direct3DSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) using (var chromaKeyEffect = MediaExtension.CreateChromaKeyEffect(inputBitmap)) { ds.Blend = CanvasBlend.Copy; ds.DrawImage(chromaKeyEffect); } }
public void ProcessFrame(ProcessVideoFrameContext context) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, context.InputFrame.Direct3DSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) { if (MediaExtension.ActiveBackgroundSet != null && MediaExtension.BackgroundSetCache != null && MediaExtension.BackgroundSetCache.TryGetValue( key: MediaExtension.ActiveBackgroundSet, value: out var set) && set.IsCompleted && set.Result.Length > 0) { try { set.Wait(); } catch (AggregateException exception) { // .NET async tasks wrap all errors in an AggregateException. // We unpack this so Win2D can directly see any lost device errors. exception.Handle(ex => { throw ex; }); } var framesPerMillisecond = MediaExtension.ActiveBackgroundFramesPerSecond / 1000; var durationInMilliseconds = set.Result.Length / framesPerMillisecond; var offsetInDuration = context.InputFrame.RelativeTime?.TotalMilliseconds % durationInMilliseconds / durationInMilliseconds ?? 0; var offsetInFrames = Convert.ToInt32(set.Result.Length * offsetInDuration); // Does someone want to fix my algorithm so this isn't necessary? var offsetInIndex = offsetInFrames <= 0 ? 0 : offsetInFrames >= set.Result.Length ? set.Result.Length - 1 : offsetInFrames; var canvas = set.Result[offsetInIndex]; var scale = inputBitmap.Bounds.Width / canvas.Bounds.Width; var bounds = canvas.Bounds; bounds.Width = bounds.Width * scale; bounds.Height = bounds.Height * scale; ds.DrawImage(canvas, bounds); } else { ds.FillRectangle(inputBitmap.Bounds, Colors.Black); } ds.DrawImage(inputBitmap); } }
public void ProcessFrame(ProcessVideoFrameContext context) { using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession()) { var saturation = new SaturationEffect() { Source = inputBitmap, Saturation = this.Saturation }; ds.DrawImage(saturation); } }
public void ProcessFrame(ProcessVideoFrameContext context) { using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(_device, context.InputFrame.Direct3DSurface)) using (CanvasRenderTarget target = CanvasRenderTarget.CreateFromDirect3D11Surface(_device, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession session = target.CreateDrawingSession()) { byte[] bytes = input.GetPixelBytes(); IntPtr ptr = Marshal.AllocHGlobal(bytes.Length); Marshal.Copy(bytes, 0, ptr, bytes.Length); Marshal.FreeHGlobal(ptr); } }
public void ProcessFrame(ProcessVideoFrameContext context) { using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession()) { ColorMatrixEffect colorMatrixEffect = new ColorMatrixEffect { Source = inputBitmap }; colorMatrixEffect.ColorMatrix = ColorMatrix; ds.DrawImage(colorMatrixEffect); } }
// </SnippetBlurAmountWin2D> // <SnippetProcessFrameWin2D> public void ProcessFrame(ProcessVideoFrameContext context) { using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession()) { var gaussianBlurEffect = new GaussianBlurEffect { Source = inputBitmap, BlurAmount = (float)BlurAmount, Optimization = EffectOptimization.Speed }; ds.DrawImage(gaussianBlurEffect); } }
public void ProcessFrame(ProcessVideoFrameContext context) { using (var output = CanvasRenderTarget.CreateFromDirect3D11Surface(device, context.OutputFrame.Direct3DSurface)) using (var input = CanvasRenderTarget.CreateFromDirect3D11Surface(device, context.InputFrame.Direct3DSurface)) { var args = new VideoEffectHandlerArgs() { ID = id, Device = device, InputFrame = input, OutputFrame = output, Properties = properties }; VideoEffectManager.ProcessFrame(args); } }
public void ProcessFrame(ProcessVideoFrameContext context) { using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = output.CreateDrawingSession()) { ds.Clear(Colors.Black); var mirrorEffect = new Transform2DEffect() { Source = input, TransformMatrix = new Matrix3x2(-1, 0, 0, 1, output.SizeInPixels.Width, 0), }; ds.DrawImage(mirrorEffect); } }
/// <summary> /// Composite the frame /// </summary> /// <param name="context">the composite frame context</param> public void CompositeFrame(CompositeVideoFrameContext context) { foreach (var surface in context.SurfacesToOverlay) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, surface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) using (var chromaKeyEffect = MediaExtension.CreateChromaKeyEffect(inputBitmap)) { var overlay = context.GetOverlayForSurface(surface); var destinationRectangle = overlay.Position; var sourceRectangle = inputBitmap.Bounds; var opacity = System.Convert.ToSingle(overlay.Opacity); ds.DrawImage(chromaKeyEffect, destinationRectangle, sourceRectangle, opacity); } } }
public void ProcessFrame(ProcessVideoFrameContext context) { using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession()) { var mills = 2000.0 - (context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value.TotalMilliseconds - 2000.0 : 0.0); var k = mills > 0 ? (float)mills / 2000.0f : 0.0f; var gaussianBlurEffect = new GaussianBlurEffect { Source = inputBitmap, BlurAmount = 30 * k, Optimization = EffectOptimization.Speed }; ds.DrawImage(gaussianBlurEffect); } }
public void ProcessFrame(ProcessVideoFrameContext context) { var inputSurface = context.InputFrame.Direct3DSurface; var outputSurface = context.OutputFrame.Direct3DSurface; using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, inputSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, outputSurface)) using (var ds = renderTarget.CreateDrawingSession()) { ds.DrawImage(inputBitmap); uint ww = inputBitmap.SizeInPixels.Width; uint hh = inputBitmap.SizeInPixels.Height; Rect rx = new Rect(0, 0, ww, hh); ds.DrawRectangle(rx, Colors.White); uint ww2 = ww - 2; uint hh2 = hh - 2; Rect rxInner = new Rect(1, 1, ww2, hh2); ds.DrawRectangle(rx, Colors.Black); } }
public void ProcessFrame(ProcessVideoFrameContext context) { using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = output.CreateDrawingSession()) { TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan(); float dispX = (float)Math.Cos(time.TotalSeconds) * 75f; float dispY = (float)Math.Sin(time.TotalSeconds) * 75f; ds.Clear(Colors.Black); var dispMap = new DisplacementMapEffect() { Source = input, XChannelSelect = EffectChannelSelect.Red, YChannelSelect = EffectChannelSelect.Green, Amount = 100f, Displacement = new Transform2DEffect() { TransformMatrix = Matrix3x2.CreateTranslation(dispX, dispY), Source = new BorderEffect() { ExtendX = CanvasEdgeBehavior.Mirror, ExtendY = CanvasEdgeBehavior.Mirror, Source = new TurbulenceEffect() { Octaves = 3 } } } }; ds.DrawImage(dispMap, -25f, -25f); } }
public void ProcessFrame(ProcessVideoFrameContext context) { try { // When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame // If we're on GPU, use InputFrame.Direct3DSurface if (context.InputFrame.SoftwareBitmap == null) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) { ds.DrawImage(inputBitmap); ds.DrawImage(this._overlay, inputBitmap.Bounds); } return; } // If we're on CPU, use InputFrame.SoftwareBitmap if (context.InputFrame.Direct3DSurface == null) { // InputFrame's raw pixels byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight]; context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer()); using (var inputBitmap = CanvasBitmap.CreateFromBytes( _canvasDevice, inputFrameBytes, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat())) using (var renderTarget = new CanvasRenderTarget( _canvasDevice, context.OutputFrame.SoftwareBitmap.PixelWidth, context.OutputFrame.SoftwareBitmap.PixelHeight, (float)context.OutputFrame.SoftwareBitmap.DpiX, context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(), CanvasAlphaMode.Premultiplied)) { using (var ds = renderTarget.CreateDrawingSession()) { ds.DrawImage(inputBitmap); ds.DrawImage(this._overlay, inputBitmap.Bounds); } } } } catch (Exception) { if (crashCount < 20) { crashCount++; Debug.WriteLine($"ProcessFrame Exception: #{crashCount}"); } else { //System.Exception HResult = 0x88990012 //Message = Objects used together must be created from the same factory instance. (Exception from HRESULT: 0x88990012) //Source = System.Private.CoreLib //StackTrace: //at System.Runtime.InteropServices.WindowsRuntime.IClosable.Close() //at System.Runtime.InteropServices.WindowsRuntime.IClosableToIDisposableAdapter.Dispose() //at VideoEffects.Win2D.OverlayVideoEffect.ProcessFrame(ProcessVideoFrameContext context) in D:\GitHub\VideoDiary\src\VideoDiary.EffectsLibrary\Win2dEffects\OverlayVideoEffect.cs:line 66 throw; } } }
public void ProcessFrame(ProcessVideoFrameContext context) { //using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) //using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) //using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession()) //{ // var effect = new EdgeDetectionEffect // { // Source = inputBitmap, // Amount = this.Amount // }; // ds.DrawImage(effect); //} // When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame // If we're on GPU, use InputFrame.Direct3DSurface if (context.InputFrame.SoftwareBitmap == null) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) { var effect = new EdgeDetectionEffect { Source = inputBitmap, Amount = this.Amount }; ds.DrawImage(effect); } return; } // If we're on CPU, use InputFrame.SoftwareBitmap if (context.InputFrame.Direct3DSurface == null) { #if DEBUG if (!debugOutputFlag) { Debug.WriteLine($"PixelFormat: {context.InputFrame.SoftwareBitmap.BitmapPixelFormat}"); debugOutputFlag = true; } #endif // ********** Test for using SoftwareBitmap.Convert ********* // //using (var inputBitmap = CanvasBitmap.CreateFromSoftwareBitmap( // _canvasDevice, // SoftwareBitmap.Convert(context.InputFrame.SoftwareBitmap, context.InputFrame.SoftwareBitmap.BitmapPixelFormat))) //PixelFormat: Bgra8 //using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) //using (var ds = renderTarget.CreateDrawingSession()) //{ // var effect = new EdgeDetectionEffect // { // Source = inputBitmap, // Amount = this.Amount // }; // ds.DrawImage(effect); //} // ********************************************************** // // InputFrame's raw pixels byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight]; context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer()); using (var inputBitmap = CanvasBitmap.CreateFromBytes( _canvasDevice, inputFrameBytes, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat())) using (var renderTarget = new CanvasRenderTarget( _canvasDevice, context.OutputFrame.SoftwareBitmap.PixelWidth, context.OutputFrame.SoftwareBitmap.PixelHeight, (float)context.OutputFrame.SoftwareBitmap.DpiX, context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(), CanvasAlphaMode.Premultiplied)) { using (var ds = renderTarget.CreateDrawingSession()) { var effect = new EdgeDetectionEffect { Source = inputBitmap, Amount = this.Amount }; ds.DrawImage(effect); } } } }
public void ProcessFrame(ProcessVideoFrameContext context) { //using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) //using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) //using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession()) //{ // var grayscale = new GrayscaleEffect() // { // Source = inputBitmap // }; // ds.DrawImage(grayscale); //} // When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame // If we're on GPU, use InputFrame.Direct3DSurface if (context.InputFrame.SoftwareBitmap == null) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) { var grayscale = new GrayscaleEffect() { Source = inputBitmap }; ds.DrawImage(grayscale); } return; } // If we're on CPU, use InputFrame.SoftwareBitmap if (context.InputFrame.Direct3DSurface == null) { // InputFrame's raw pixels byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight]; context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer()); using (var inputBitmap = CanvasBitmap.CreateFromBytes( _canvasDevice, inputFrameBytes, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat())) using (var renderTarget = new CanvasRenderTarget( _canvasDevice, context.OutputFrame.SoftwareBitmap.PixelWidth, context.OutputFrame.SoftwareBitmap.PixelHeight, (float)context.OutputFrame.SoftwareBitmap.DpiX, context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(), CanvasAlphaMode.Premultiplied)) { using (var ds = renderTarget.CreateDrawingSession()) { var grayscale = new GrayscaleEffect() { Source = inputBitmap }; ds.DrawImage(grayscale); } } } }
//private bool correctionFlag = false; public void ProcessFrame(ProcessVideoFrameContext context) { using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession()) using (var scaleEffect = new ScaleEffect()) using (CanvasSolidColorBrush solidColorBrush = new CanvasSolidColorBrush(_canvasDevice, _backgroundColor)) { solidColorBrush.Opacity = _backgroundOpacity; double rel = context.InputFrame.RelativeTime.Value.Ticks / (double)TimeSpan.TicksPerMillisecond; //context.OutputFrame.Duration = new TimeSpan( (long)(frameLength * TimeSpan.TicksPerMillisecond)); int frameTimeCounter = (int)Math.Round(rel / _frameLength, 0); int[] pitch = new int[_count]; int[] yaw = new int[_count]; int[] fov = new int[_count]; for (int i = 0; i < _count; i++) { try { //pitch[i] = this.pitch[ (frameTimeCounter + (int)Math.Round(offset, 0)) * (count) + i]; //fov[i] = this.fov[ (frameTimeCounter + (int)Math.Round(offset, 0)) * (count) + i]; //yaw[i] = this.yaw[ (frameTimeCounter + (int)Math.Round(offset, 0)) * (count) + i]; pitch[i] = this._pitch[(frameTimeCounter + (int)_offset) * (_count) + i]; fov[i] = this._fov[(frameTimeCounter + (int)_offset) * (_count) + i]; yaw[i] = this._yaw[(frameTimeCounter + (int)_offset) * (_count) + i]; } catch (ArgumentOutOfRangeException ex) { Debug.WriteLine(ex.Message); pitch[i] = 0; fov[i] = 0; yaw[i] = 0; } } byte[] tab = Heatmap.GenerateHeatmap(pitch, yaw, fov); CanvasBitmap cb = CanvasBitmap.CreateFromBytes(_canvasDevice, tab, 64, 64, Windows.Graphics.DirectX.DirectXPixelFormat.B8G8R8A8UIntNormalized, 96, CanvasAlphaMode.Premultiplied); scaleEffect.Source = cb; scaleEffect.Scale = new System.Numerics.Vector2((float)_width / 64, (float)_height / 64); scaleEffect.InterpolationMode = CanvasImageInterpolation.Cubic; scaleEffect.BorderMode = EffectBorderMode.Hard; if (_graysclaleVideoFlag) { var grayScaleEffect = new GrayscaleEffect { BufferPrecision = CanvasBufferPrecision.Precision8UIntNormalized, CacheOutput = false, Source = inputBitmap }; ds.DrawImage(grayScaleEffect); } else { ds.DrawImage(inputBitmap); } ds.DrawImage(scaleEffect, 0, 0, new Windows.Foundation.Rect { Height = _height, Width = _width }, _heatmapOpacity); if (_generateDots) { for (int i = 0; i < _count; i++) { ds.FillCircle(yaw[i] * _width / 64, pitch[i] * _height / 64, _dotsRadius, _colors[i % 5]); } } ds.FillRectangle(new Windows.Foundation.Rect { Height = _height, Width = _width }, solidColorBrush); ds.Flush(); } }
public async void ProcessFrame(ProcessVideoFrameContext context) { // When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame // If we're on GPU, use InputFrame.Direct3DSurface if (context.InputFrame.SoftwareBitmap == null) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) { var chromaKeyEffect = new ChromaKeyEffect { Color = ChromaColor, Source = inputBitmap, Tolerance = Tolerance, Feather = true }; //only load bg image if it hasnt already been loaded or if image is different if (_backgroundBitmap == null || _lastUsedImageFilePath != _imageFileName) { _backgroundBitmap = await CanvasBitmap.LoadAsync(ds, new Uri($"ms-appdata:///Local/{ImageFileName}")); } _lastUsedImageFilePath = _imageFileName; //keep track of any incoming image changes var compositeEffect = new CompositeEffect { Sources = { _backgroundBitmap, chromaKeyEffect } }; ds.DrawImage(compositeEffect); } return; } // If we're on CPU, use InputFrame.SoftwareBitmap if (context.InputFrame.Direct3DSurface == null) { // InputFrame's raw pixels byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight]; context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer()); using (var inputBitmap = CanvasBitmap.CreateFromBytes( _canvasDevice, inputFrameBytes, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat())) using (var renderTarget = new CanvasRenderTarget( _canvasDevice, context.OutputFrame.SoftwareBitmap.PixelWidth, context.OutputFrame.SoftwareBitmap.PixelHeight, (float)context.OutputFrame.SoftwareBitmap.DpiX, context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(), CanvasAlphaMode.Ignore)) { using (var ds = renderTarget.CreateDrawingSession()) { var chroma = new ChromaKeyEffect { Color = ChromaColor, Source = inputBitmap, Tolerance = Tolerance }; //only load bg image if it hasnt already been loaded or if image is different if (_backgroundBitmap == null || _lastUsedImageFilePath != _imageFileName) { _backgroundBitmap = await CanvasBitmap.LoadAsync(ds, new Uri($"ms-appdata:///Local/{ImageFileName}")); } //keep track of any incoming image changes _lastUsedImageFilePath = _imageFileName; var compositeEffect = new CompositeEffect { Mode = CanvasComposite.Add }; compositeEffect.Sources.Add(_backgroundBitmap); compositeEffect.Sources.Add(chroma); ds.DrawImage(compositeEffect); } } } }
public void ProcessFrame(ProcessVideoFrameContext context) { var inputDirect3DSurface = context.InputFrame.Direct3DSurface; float anyDpi = 96.0f; var width = context.OutputFrame.Direct3DSurface.Description.Width; var height = context.OutputFrame.Direct3DSurface.Description.Height; // Part 1: Apply effect using Imaging SDK CanvasRenderTarget renderTarget = new CanvasRenderTarget(canvasDevice, (float)width, (float)height, anyDpi); if (inputDirect3DSurface != null && renderTarget != null) { if (m_direct3DSurfaceImageSource == null) { m_direct3DSurfaceImageSource = new Direct3DSurfaceImageSource(); } m_direct3DSurfaceImageSource.Direct3DSurface = inputDirect3DSurface; ((IImageConsumer)m_grayScaleEffect).Source = m_direct3DSurfaceImageSource; if (m_direct3DSurfaceRenderer == null) { m_direct3DSurfaceRenderer = new Direct3DSurfaceRenderer(); } m_direct3DSurfaceRenderer.Direct3DSurface = renderTarget; m_direct3DSurfaceRenderer.Source = m_grayScaleEffect; var task = m_direct3DSurfaceRenderer.RenderAsync().AsTask(); task.Wait(); m_direct3DSurfaceRenderer.Direct3DSurface = null; m_direct3DSurfaceImageSource.Direct3DSurface = null; } // Part 2: Apply another effect using Win2D var size = context.OutputFrame.Direct3DSurface.Description.Width * context.OutputFrame.Direct3DSurface.Description.Height; var colors = Enumerable.Repeat(Colors.Black, size).ToArray(); var inputBitmap = CanvasBitmap.CreateFromColors(canvasDevice, colors, width, height, anyDpi); inputBitmap.CopyPixelsFromBitmap(renderTarget); using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = output.CreateDrawingSession()) { TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan(); float dispX = (float)Math.Cos(time.TotalSeconds) * 75f; float dispY = (float)Math.Sin(time.TotalSeconds) * 75f; ds.Clear(Colors.Black); var dispMap = new DisplacementMapEffect() { Source = inputBitmap, XChannelSelect = EffectChannelSelect.Red, YChannelSelect = EffectChannelSelect.Green, Amount = 100f, Displacement = new Transform2DEffect() { TransformMatrix = Matrix3x2.CreateTranslation(dispX, dispY), Source = new BorderEffect() { ExtendX = CanvasEdgeBehavior.Mirror, ExtendY = CanvasEdgeBehavior.Mirror, Source = new TurbulenceEffect() { Octaves = 3 } } } }; ds.DrawImage(dispMap, -25f, -25f); } ((IDisposable)renderTarget).Dispose(); ((IDisposable)inputBitmap).Dispose(); }
public void ProcessFrame(ProcessVideoFrameContext context) { CanvasRenderTarget rt = new CanvasRenderTarget(canvasDevice, context.OutputFrame.Direct3DSurface.Description.Width, context.OutputFrame.Direct3DSurface.Description.Height, 96.0f); using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = rt.CreateDrawingSession()) { TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan(); float dispX = (float)Math.Cos(time.TotalSeconds) * 75f; float dispY = (float)Math.Sin(time.TotalSeconds) * 75f; ds.Clear(Colors.Black); //var posterizeEffect = new PosterizeEffect() //{ // Source = input, // BlueValueCount = 2, // RedValueCount = 2, // GreenValueCount = 2, //}; var transformEffect = new Transform2DEffect() { Source = input, TransformMatrix = new System.Numerics.Matrix3x2(-1, 0, 0, 1, rt.SizeInPixels.Width, 0), }; var discreteTransferEffect = new DiscreteTransferEffect() { Source = transformEffect, RedTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f }, GreenTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f }, BlueTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f }, AlphaDisable = true, }; var dispMap = new ConvolveMatrixEffect() { KernelMatrix = new float[] { 0, +1, 0, +1, -4, +1, 0, +1, 0, }, Source = discreteTransferEffect, Divisor = 0.2f, }; var modEffect = new ArithmeticCompositeEffect() { Source1 = dispMap, Source2 = dispMap, MultiplyAmount = 1, }; //var finalPosterizeEffect = new PosterizeEffect() //{ // Source = modEffect, // BlueValueCount = 2, // RedValueCount = 2, // GreenValueCount = 2, //}; //var dispMap = new EdgeDetectionEffect() //{ // Source = greyScaleEffect, // Mode = EdgeDetectionEffectMode.Sobel, //}; Rect src = new Rect(rt.SizeInPixels.Width / 3, rt.SizeInPixels.Height / 3, rt.SizeInPixels.Width / 3, rt.SizeInPixels.Height / 3); //ds.DrawImage(input, bottomLeft); //ds.DrawImage(discreteTransferEffect, bottomRight, src); //ds.DrawImage(dispMap, topLeft, src); ds.DrawImage(modEffect, src, src); } int centerX = 0; int centerY = 0; int cubletWidth = 0; int cubletHeight = 0; int analysisAreaX = (int)rt.SizeInPixels.Width * 3 / 10; int analysisWidth = (int)rt.SizeInPixels.Width * 4 / 10; byte[] analysisHorzBytes = rt.GetPixelBytes(analysisAreaX, (int)rt.SizeInPixels.Height / 2, analysisWidth, 1); int analysisAreaY = (int)rt.SizeInPixels.Height * 3 / 10; int analysisHeight = (int)rt.SizeInPixels.Height * 4 / 10; byte[] analysisVertBytes = rt.GetPixelBytes((int)rt.SizeInPixels.Width / 2, analysisAreaY, 1, analysisHeight); int foundLeft = 0; for (int i = 0; i < analysisWidth / 2; i++) { byte b = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 0]; byte g = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 1]; byte r = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 2]; if ((r > 100 || g > 100 || b > 50) && foundLeft == 0) { foundLeft = i; } } int foundRight = 0; for (int i = 0; i < analysisWidth / 2; i++) { byte r = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 0]; byte g = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 1]; byte b = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 2]; if ((r > 100 || g > 100 || b > 50) && foundRight == 0) { foundRight = i; } } int foundTop = 0; for (int i = 0; i < analysisHeight / 2; i++) { byte r = analysisVertBytes[4 * (analysisHeight / 2 - i) + 0]; byte g = analysisVertBytes[4 * (analysisHeight / 2 - i) + 1]; byte b = analysisVertBytes[4 * (analysisHeight / 2 - i) + 2]; if ((r > 100 || g > 100 || b > 50) && foundTop == 0) { foundTop = i; } } int foundBottom = 0; for (int i = 0; i < analysisHeight / 2; i++) { byte r = analysisVertBytes[4 * (analysisHeight / 2 + i) + 0]; byte g = analysisVertBytes[4 * (analysisHeight / 2 + i) + 1]; byte b = analysisVertBytes[4 * (analysisHeight / 2 + i) + 2]; if ((r > 100 || g > 100 || b > 50) && foundBottom == 0) { foundBottom = i; } } centerX = (-foundLeft + foundRight) / 2 + (int)rt.SizeInPixels.Width / 2; centerY = (-foundTop + foundBottom) / 2 + (int)rt.SizeInPixels.Height / 2; cubletWidth = (int)((foundLeft + foundRight) * 1.2f); cubletHeight = (int)((foundTop + foundBottom) * 1.2f); // No 2d arrays in WinRT components? Boo. // "Error C1113 #using failed on 'rubikscuberecognitionwinrt.winmd' RubiksCubeRecognitionLib Vector2[] cubletCenters = new Vector2[3 * 3]; cubletCenters[1 * 3 + 1] = new Vector2(centerX, centerY); cubletCenters[1 * 3 + 0] = new Vector2(centerX, centerY - cubletHeight); cubletCenters[1 * 3 + 2] = new Vector2(centerX, centerY + cubletHeight); cubletCenters[0 * 3 + 1] = new Vector2(centerX - cubletWidth, centerY); cubletCenters[0 * 3 + 0] = new Vector2(centerX - cubletWidth, centerY - cubletHeight); cubletCenters[0 * 3 + 2] = new Vector2(centerX - cubletWidth, centerY + cubletHeight); cubletCenters[2 * 3 + 1] = new Vector2(centerX + cubletWidth, centerY); cubletCenters[2 * 3 + 0] = new Vector2(centerX + cubletWidth, centerY - cubletHeight); cubletCenters[2 * 3 + 2] = new Vector2(centerX + cubletWidth, centerY + cubletHeight); cubeletColorsMutex.WaitOne(); using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface)) using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface)) using (CanvasDrawingSession ds = output.CreateDrawingSession()) { var transformEffect = new Transform2DEffect() { Source = input, TransformMatrix = new System.Numerics.Matrix3x2(-1, 0, 0, 1, output.SizeInPixels.Width, 0), }; Rect src = new Rect(0, 0, output.SizeInPixels.Width, output.SizeInPixels.Height); ds.DrawImage(transformEffect, src, src); // Draw a crosshair on the screen ds.FillRectangle(new Rect(output.SizeInPixels.Width / 2 - 3, output.SizeInPixels.Height / 4, 6, output.SizeInPixels.Height / 2), Colors.Gray); ds.FillRectangle(new Rect(output.SizeInPixels.Width / 4, output.SizeInPixels.Height / 2 - 3, output.SizeInPixels.Width / 2, 6), Colors.Gray); if (true) { for (int x = 0; x < 3; x++) { for (int y = 0; y < 3; y++) { int sampleWidth = 2; byte[] cubletBytes = input.GetPixelBytes((int)cubletCenters[(2 - x) * 3 + y].X - sampleWidth / 2, (int)cubletCenters[(2 - x) * 3 + y].Y - sampleWidth / 2, sampleWidth, sampleWidth); int totalR = 0; int totalG = 0; int totalB = 0; for (int i = 0; i < sampleWidth * sampleWidth; i++) { totalB += cubletBytes[4 * i + 0]; totalG += cubletBytes[4 * i + 1]; totalR += cubletBytes[4 * i + 2]; } cubletColors[x * 3 + y] = Color.FromArgb(255, (byte)(totalR / (sampleWidth * sampleWidth)), (byte)(totalG / (sampleWidth * sampleWidth)), (byte)(totalB / (sampleWidth * sampleWidth))); } } for (int x = 0; x < 3; x++) { for (int y = 0; y < 3; y++) { ds.FillRectangle((float)cubletCenters[x * 3 + y].X - 12, (float)cubletCenters[x * 3 + y].Y - 12, 24, 24, Colors.Black); ds.FillRectangle((float)cubletCenters[x * 3 + y].X - 10, (float)cubletCenters[x * 3 + y].Y - 10, 20, 20, cubletColors[x * 3 + y]); } } } } cubeletColorsMutex.ReleaseMutex(); }
public void ProcessFrame(ProcessVideoFrameContext context) { //using (var input = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) //using (var output = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) //using (var ds = output.CreateDrawingSession()) //{ // var time = context.InputFrame.RelativeTime ?? new TimeSpan(); // ds.Clear(Colors.Black); // for (uint i = 0; i < _numColumns; i++) // { // for (uint j = 0; j < _numRows; j++) // { // _crops[i, j].Source = input; // float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f); // float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j); // Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile); // _transforms[i, j].TransformMatrix = // Matrix3x2.CreateRotation(rotation, centerPoint) * // Matrix3x2.CreateScale(scale, centerPoint); // ds.DrawImage(_transforms[i, j]); // } // } //} // When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame // If we're on GPU, use InputFrame.Direct3DSurface if (context.InputFrame.SoftwareBitmap == null) { using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface)) using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface)) using (var ds = renderTarget.CreateDrawingSession()) { var time = context.InputFrame.RelativeTime ?? new TimeSpan(); ds.Clear(Colors.Black); for (uint i = 0; i < _numColumns; i++) { for (uint j = 0; j < _numRows; j++) { _crops[i, j].Source = inputBitmap; float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f); float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j); Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile); _transforms[i, j].TransformMatrix = Matrix3x2.CreateRotation(rotation, centerPoint) * Matrix3x2.CreateScale(scale, centerPoint); ds.DrawImage(_transforms[i, j]); } } } return; } // If we're on CPU, use InputFrame.SoftwareBitmap if (context.InputFrame.Direct3DSurface == null) { // InputFrame's raw pixels byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight]; context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer()); using (var inputBitmap = CanvasBitmap.CreateFromBytes( _canvasDevice, inputFrameBytes, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat())) using (var renderTarget = new CanvasRenderTarget( _canvasDevice, context.OutputFrame.SoftwareBitmap.PixelWidth, context.OutputFrame.SoftwareBitmap.PixelHeight, (float)context.OutputFrame.SoftwareBitmap.DpiX, context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(), CanvasAlphaMode.Premultiplied)) { using (var ds = renderTarget.CreateDrawingSession()) { var time = context.InputFrame.RelativeTime ?? new TimeSpan(); ds.Clear(Colors.Black); for (uint i = 0; i < _numColumns; i++) { for (uint j = 0; j < _numRows; j++) { _crops[i, j].Source = inputBitmap; float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f); float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j); Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile); _transforms[i, j].TransformMatrix = Matrix3x2.CreateRotation(rotation, centerPoint) * Matrix3x2.CreateScale(scale, centerPoint); ds.DrawImage(_transforms[i, j]); } } } } } }