public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
ds.Clear(Colors.Black);
for (uint i = 0; i < numColumns; i++)
{
for (uint j = 0; j < numRows; j++)
{
crops[i, j].Source = input;
float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
Vector2 centerPoint = new Vector2((i + 0.5f) * pixelsPerTile, (j + 0.5f) * pixelsPerTile);
transforms[i, j].TransformMatrix =
Matrix3x2.CreateRotation(rotation, centerPoint) *
Matrix3x2.CreateScale(scale, centerPoint);
ds.DrawImage(transforms[i, j]);
}
}
}
}
public void ProcessFrameGPU(ProcessVideoFrameContext context)
{
var inputSurface = context.InputFrame.Direct3DSurface;
var outputSurface = context.OutputFrame.Direct3DSurface;
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(m_canvasDevice, inputSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(m_canvasDevice, outputSurface))
using (var drawingSession = renderTarget.CreateDrawingSession())
{
var normalize = new Matrix5x4()
{
M11 = 1f, M22 = 1f, M33 = 1f, M44 = 1f, M51 = -m_mean.X, M52 = -m_mean.Y, M53 = -m_mean.Z
};
var normalize2 = new Matrix5x4()
{
M11 = 1 / m_std.X, M22 = 1 / m_std.Y, M33 = 1 / m_std.Z, M44 = 1.0f
};
// https://microsoft.github.io/Win2D/html/T_Microsoft_Graphics_Canvas_Effects_ColorMatrixEffect.htm
var PreprocessTransfrom = new ColorMatrixEffect()
{
ColorMatrix = normalize2,
Source = new ColorMatrixEffect()
{
ColorMatrix = normalize,
Source = inputBitmap
}
};
drawingSession.DrawImage(PreprocessTransfrom);
}
}
public void CompositeFrame(CompositeVideoFrameContext context)
{
IDirect3DSurface outputSurface = context.OutputFrame.Direct3DSurface;
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, outputSurface))
using (CanvasDrawingSession drawSession = renderTarget.CreateDrawingSession())
{
foreach (var overlaySurface in context.SurfacesToOverlay)
{
var overlay = context.GetOverlayForSurface(overlaySurface);
var width = (float)overlay.Position.Width;
var height = (float)overlay.Position.Height;
using (var overlayBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, overlaySurface))
using (var videoBrush = new CanvasImageBrush(_canvasDevice, overlayBitmap))
{
var scale = width / overlay.Clip.GetVideoEncodingProperties().Width;
videoBrush.Transform = Matrix3x2.CreateScale(scale) * Matrix3x2.CreateTranslation((float)overlay.Position.X, (float)overlay.Position.Y);
drawSession.FillEllipse(new Vector2((float)overlay.Position.X + width / 2, (float)overlay.Position.Y + height / 2), width / 2, height / 2, videoBrush);
}
}
drawSession.DrawText("Party\nTime!", new Vector2(_backgroundProperties.Width / 1.5f, 100), Windows.UI.Colors.CornflowerBlue,
new CanvasTextFormat()
{
FontSize = (float)_backgroundProperties.Width / 13,
FontWeight = new FontWeight()
{
Weight = 999
},
HorizontalAlignment = CanvasHorizontalAlignment.Center,
VerticalAlignment = CanvasVerticalAlignment.Center
});
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputSurface = context.InputFrame.Direct3DSurface;
var outputSurface = context.OutputFrame.Direct3DSurface;
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, inputSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, outputSurface))
using (var ds = renderTarget.CreateDrawingSession())
using (var brush = new CanvasImageBrush(canvasDevice, inputBitmap))
using (var textCommandList = new CanvasCommandList(canvasDevice))
{
using (var clds = textCommandList.CreateDrawingSession())
{
clds.DrawText(
"Win2D\nMediaClip",
(float)inputBitmap.Size.Width / 2,
(float)inputBitmap.Size.Height / 2,
brush,
new CanvasTextFormat()
{
FontSize = (float)inputBitmap.Size.Width / 5,
FontWeight = new FontWeight()
{
Weight = 999
},
HorizontalAlignment = CanvasHorizontalAlignment.Center,
VerticalAlignment = CanvasVerticalAlignment.Center
});
}
var background = new GaussianBlurEffect()
{
BlurAmount = 10,
BorderMode = EffectBorderMode.Hard,
Source = new BrightnessEffect()
{
BlackPoint = new Vector2(0.5f, 0.7f),
Source = new SaturationEffect()
{
Saturation = 0,
Source = inputBitmap
}
}
};
var shadow = new ShadowEffect()
{
Source = textCommandList,
BlurAmount = 10
};
var composite = new CompositeEffect()
{
Sources = { background, shadow, textCommandList }
};
ds.DrawImage(composite);
}
}
// ** Methods ** //
// This is run for every video frame passed in the media pipeline (MediaPlayer, MediaCapture, etc)
public void ProcessFrame(ProcessVideoFrameContext context)
{
videoFrameToProcess = VideoFrame.CreateWithDirect3D11Surface(context.InputFrame.Direct3DSurface);
// ********** Draw Bounding Boxes with Win2D ********** //
// Use Direct3DSurface if using GPU memory
if (context.InputFrame.Direct3DSurface != null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
foreach (var box in filteredBoxes)
{
var x = (uint)Math.Max(box.X, 0);
var y = (uint)Math.Max(box.Y, 0);
var w = (uint)Math.Min(renderTarget.Bounds.Width - x, box.Width);
var h = (uint)Math.Min(renderTarget.Bounds.Height - y, box.Height);
// Draw the Text 10px above the top of the bounding box
ds.DrawText(box.Label, x, y - 10, Colors.Yellow);
ds.DrawRectangle(new Rect(x, y, w, h), new CanvasSolidColorBrush(canvasDevice, Colors.Yellow), 2f);
}
}
return;
}
// Use SoftwareBitmap if using CPU memory
if (context.InputFrame.SoftwareBitmap != null)
{
// InputFrame's pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(canvasDevice, inputFrameBytes, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(canvasDevice, context.OutputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, (float)context.OutputFrame.SoftwareBitmap.DpiX, context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(), CanvasAlphaMode.Premultiplied))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
foreach (var box in filteredBoxes)
{
var x = (uint)Math.Max(box.X, 0);
var y = (uint)Math.Max(box.Y, 0);
var w = (uint)Math.Min(context.OutputFrame.SoftwareBitmap.PixelWidth - x, box.Width);
var h = (uint)Math.Min(context.OutputFrame.SoftwareBitmap.PixelHeight - y, box.Height);
// Draw the Text 10px above the top of the bounding box
ds.DrawText(box.Label, x, y - 10, Colors.Yellow);
ds.DrawRectangle(new Rect(x, y, w, h), new CanvasSolidColorBrush(canvasDevice, Colors.Yellow), 2f);
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
// If memory type is CPU, the frame is in InputFrame.SoftwareBitmap.
// For GPU, the frame is in InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var sepia = new SepiaEffect
{
Source = inputBitmap,
Intensity = this.Intensity
};
ds.DrawImage(sepia);
}
return;
}
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var sepia = new SepiaEffect
{
Source = inputBitmap,
Intensity = this.Intensity
};
ds.DrawImage(sepia);
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.Transform = MediaExtension.CreateTransform(inputBitmap);
ds.DrawImage(inputBitmap);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
bool skipMaskPred = false;
if (context.InputFrame.IsDiscontinuous)
{
streamStartDelta = TimeSpan.FromTicks(DateTime.Now.Ticks) - context.InputFrame.SystemRelativeTime.Value;
}
else
{
if ((TimeSpan.FromTicks(DateTime.Now.Ticks) - context.InputFrame.SystemRelativeTime.Value - streamStartDelta) > TimeSpan.FromMilliseconds(maxDelay))
{
skipMaskPred = true;
}
}
if (!skipMaskPred)
{
frameCount++;
features["0"] = context.InputFrame;
var resTask = _learningModel.EvaluateFeaturesAsync(features, string.Empty).AsTask();
var startTime = DateTime.Now.Ticks;
resTask.Wait();
Debug.WriteLine("delta {0}", TimeSpan.FromTicks(DateTime.Now.Ticks - startTime));
}
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasBitmap inputMask = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, output.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
ds.Clear(Colors.Green);
var addAlpha = new ColorMatrixEffect()
{
Source = inputMask,
ColorMatrix = RToAlpha
};
var resize = new ScaleEffect()
{
Source = addAlpha,
Scale = new Vector2(((float)inputBitmap.SizeInPixels.Width / inputMask.SizeInPixels.Width), ((float)inputBitmap.SizeInPixels.Height / inputMask.SizeInPixels.Height))
};
var blend = new AlphaMaskEffect()
{
Source = inputBitmap,
AlphaMask = resize
};
ds.DrawImage(blend);
ds.DrawText(String.Format("FPS: {0:f1}", currentFPS), fpsLabelOffset, fpsLabelColor);
}
}
//private ICanvasImage[] BackgroundImages { get; set; } = default(ICanvasImage[]);
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
using (var chromaKeyEffect = MediaExtension.CreateChromaKeyEffect(inputBitmap))
{
ds.Blend = CanvasBlend.Copy;
ds.DrawImage(chromaKeyEffect);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
if (MediaExtension.ActiveBackgroundSet != null &&
MediaExtension.BackgroundSetCache != null &&
MediaExtension.BackgroundSetCache.TryGetValue(
key: MediaExtension.ActiveBackgroundSet, value: out var set) &&
set.IsCompleted &&
set.Result.Length > 0)
{
try
{
set.Wait();
}
catch (AggregateException exception)
{
// .NET async tasks wrap all errors in an AggregateException.
// We unpack this so Win2D can directly see any lost device errors.
exception.Handle(ex => { throw ex; });
}
var framesPerMillisecond = MediaExtension.ActiveBackgroundFramesPerSecond / 1000;
var durationInMilliseconds = set.Result.Length / framesPerMillisecond;
var offsetInDuration = context.InputFrame.RelativeTime?.TotalMilliseconds % durationInMilliseconds / durationInMilliseconds ?? 0;
var offsetInFrames = Convert.ToInt32(set.Result.Length * offsetInDuration);
// Does someone want to fix my algorithm so this isn't necessary?
var offsetInIndex = offsetInFrames <= 0 ? 0
: offsetInFrames >= set.Result.Length ? set.Result.Length - 1
: offsetInFrames;
var canvas = set.Result[offsetInIndex];
var scale = inputBitmap.Bounds.Width / canvas.Bounds.Width;
var bounds = canvas.Bounds;
bounds.Width = bounds.Width * scale;
bounds.Height = bounds.Height * scale;
ds.DrawImage(canvas, bounds);
}
else
{
ds.FillRectangle(inputBitmap.Bounds, Colors.Black);
}
ds.DrawImage(inputBitmap);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
var saturation = new SaturationEffect()
{
Source = inputBitmap,
Saturation = this.Saturation
};
ds.DrawImage(saturation);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(_device, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget target = CanvasRenderTarget.CreateFromDirect3D11Surface(_device, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession session = target.CreateDrawingSession())
{
byte[] bytes = input.GetPixelBytes();
IntPtr ptr = Marshal.AllocHGlobal(bytes.Length);
Marshal.Copy(bytes, 0, ptr, bytes.Length);
Marshal.FreeHGlobal(ptr);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
ColorMatrixEffect colorMatrixEffect = new ColorMatrixEffect
{
Source = inputBitmap
};
colorMatrixEffect.ColorMatrix = ColorMatrix;
ds.DrawImage(colorMatrixEffect);
}
}
// </SnippetBlurAmountWin2D>
// <SnippetProcessFrameWin2D>
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
var gaussianBlurEffect = new GaussianBlurEffect
{
Source = inputBitmap,
BlurAmount = (float)BlurAmount,
Optimization = EffectOptimization.Speed
};
ds.DrawImage(gaussianBlurEffect);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (var output = CanvasRenderTarget.CreateFromDirect3D11Surface(device, context.OutputFrame.Direct3DSurface))
using (var input = CanvasRenderTarget.CreateFromDirect3D11Surface(device, context.InputFrame.Direct3DSurface))
{
var args = new VideoEffectHandlerArgs()
{
ID = id,
Device = device,
InputFrame = input,
OutputFrame = output,
Properties = properties
};
VideoEffectManager.ProcessFrame(args);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
ds.Clear(Colors.Black);
var mirrorEffect = new Transform2DEffect()
{
Source = input,
TransformMatrix = new Matrix3x2(-1, 0, 0, 1, output.SizeInPixels.Width, 0),
};
ds.DrawImage(mirrorEffect);
}
}
/// <summary>
/// Composite the frame
/// </summary>
/// <param name="context">the composite frame context</param>
public void CompositeFrame(CompositeVideoFrameContext context)
{
foreach (var surface in context.SurfacesToOverlay)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, surface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
using (var chromaKeyEffect = MediaExtension.CreateChromaKeyEffect(inputBitmap))
{
var overlay = context.GetOverlayForSurface(surface);
var destinationRectangle = overlay.Position;
var sourceRectangle = inputBitmap.Bounds;
var opacity = System.Convert.ToSingle(overlay.Opacity);
ds.DrawImage(chromaKeyEffect, destinationRectangle, sourceRectangle, opacity);
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
var mills = 2000.0 - (context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value.TotalMilliseconds - 2000.0 : 0.0);
var k = mills > 0 ? (float)mills / 2000.0f : 0.0f;
var gaussianBlurEffect = new GaussianBlurEffect
{
Source = inputBitmap,
BlurAmount = 30 * k,
Optimization = EffectOptimization.Speed
};
ds.DrawImage(gaussianBlurEffect);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputSurface = context.InputFrame.Direct3DSurface;
var outputSurface = context.OutputFrame.Direct3DSurface;
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, inputSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, outputSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
uint ww = inputBitmap.SizeInPixels.Width;
uint hh = inputBitmap.SizeInPixels.Height;
Rect rx = new Rect(0, 0, ww, hh);
ds.DrawRectangle(rx, Colors.White);
uint ww2 = ww - 2;
uint hh2 = hh - 2;
Rect rxInner = new Rect(1, 1, ww2, hh2);
ds.DrawRectangle(rx, Colors.Black);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
float dispX = (float)Math.Cos(time.TotalSeconds) * 75f;
float dispY = (float)Math.Sin(time.TotalSeconds) * 75f;
ds.Clear(Colors.Black);
var dispMap = new DisplacementMapEffect()
{
Source = input,
XChannelSelect = EffectChannelSelect.Red,
YChannelSelect = EffectChannelSelect.Green,
Amount = 100f,
Displacement = new Transform2DEffect()
{
TransformMatrix = Matrix3x2.CreateTranslation(dispX, dispY),
Source = new BorderEffect()
{
ExtendX = CanvasEdgeBehavior.Mirror,
ExtendY = CanvasEdgeBehavior.Mirror,
Source = new TurbulenceEffect()
{
Octaves = 3
}
}
}
};
ds.DrawImage(dispMap, -25f, -25f);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
try
{
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
ds.DrawImage(this._overlay, inputBitmap.Bounds);
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
ds.DrawImage(this._overlay, inputBitmap.Bounds);
}
}
}
}
catch (Exception)
{
if (crashCount < 20)
{
crashCount++;
Debug.WriteLine($"ProcessFrame Exception: #{crashCount}");
}
else
{
//System.Exception HResult = 0x88990012
//Message = Objects used together must be created from the same factory instance. (Exception from HRESULT: 0x88990012)
//Source = System.Private.CoreLib
//StackTrace:
//at System.Runtime.InteropServices.WindowsRuntime.IClosable.Close()
//at System.Runtime.InteropServices.WindowsRuntime.IClosableToIDisposableAdapter.Dispose()
//at VideoEffects.Win2D.OverlayVideoEffect.ProcessFrame(ProcessVideoFrameContext context) in D:\GitHub\VideoDiary\src\VideoDiary.EffectsLibrary\Win2dEffects\OverlayVideoEffect.cs:line 66
throw;
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
//using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
//using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
//{
// var effect = new EdgeDetectionEffect
// {
// Source = inputBitmap,
// Amount = this.Amount
// };
// ds.DrawImage(effect);
//}
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var effect = new EdgeDetectionEffect
{
Source = inputBitmap,
Amount = this.Amount
};
ds.DrawImage(effect);
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
#if DEBUG
if (!debugOutputFlag)
{
Debug.WriteLine($"PixelFormat: {context.InputFrame.SoftwareBitmap.BitmapPixelFormat}");
debugOutputFlag = true;
}
#endif
// ********** Test for using SoftwareBitmap.Convert ********* //
//using (var inputBitmap = CanvasBitmap.CreateFromSoftwareBitmap(
// _canvasDevice,
// SoftwareBitmap.Convert(context.InputFrame.SoftwareBitmap, context.InputFrame.SoftwareBitmap.BitmapPixelFormat))) //PixelFormat: Bgra8
//using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (var ds = renderTarget.CreateDrawingSession())
//{
// var effect = new EdgeDetectionEffect
// {
// Source = inputBitmap,
// Amount = this.Amount
// };
// ds.DrawImage(effect);
//}
// ********************************************************** //
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var effect = new EdgeDetectionEffect
{
Source = inputBitmap,
Amount = this.Amount
};
ds.DrawImage(effect);
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
//using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
//using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
//{
// var grayscale = new GrayscaleEffect()
// {
// Source = inputBitmap
// };
// ds.DrawImage(grayscale);
//}
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var grayscale = new GrayscaleEffect()
{
Source = inputBitmap
};
ds.DrawImage(grayscale);
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var grayscale = new GrayscaleEffect()
{
Source = inputBitmap
};
ds.DrawImage(grayscale);
}
}
}
}
//private bool correctionFlag = false;
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
using (var scaleEffect = new ScaleEffect())
using (CanvasSolidColorBrush solidColorBrush = new CanvasSolidColorBrush(_canvasDevice, _backgroundColor))
{
solidColorBrush.Opacity = _backgroundOpacity;
double rel = context.InputFrame.RelativeTime.Value.Ticks / (double)TimeSpan.TicksPerMillisecond;
//context.OutputFrame.Duration = new TimeSpan( (long)(frameLength * TimeSpan.TicksPerMillisecond));
int frameTimeCounter = (int)Math.Round(rel / _frameLength, 0);
int[] pitch = new int[_count];
int[] yaw = new int[_count];
int[] fov = new int[_count];
for (int i = 0; i < _count; i++)
{
try
{
//pitch[i] = this.pitch[ (frameTimeCounter + (int)Math.Round(offset, 0)) * (count) + i];
//fov[i] = this.fov[ (frameTimeCounter + (int)Math.Round(offset, 0)) * (count) + i];
//yaw[i] = this.yaw[ (frameTimeCounter + (int)Math.Round(offset, 0)) * (count) + i];
pitch[i] = this._pitch[(frameTimeCounter + (int)_offset) * (_count) + i];
fov[i] = this._fov[(frameTimeCounter + (int)_offset) * (_count) + i];
yaw[i] = this._yaw[(frameTimeCounter + (int)_offset) * (_count) + i];
}
catch (ArgumentOutOfRangeException ex)
{
Debug.WriteLine(ex.Message);
pitch[i] = 0;
fov[i] = 0;
yaw[i] = 0;
}
}
byte[] tab = Heatmap.GenerateHeatmap(pitch, yaw, fov);
CanvasBitmap cb = CanvasBitmap.CreateFromBytes(_canvasDevice, tab, 64, 64, Windows.Graphics.DirectX.DirectXPixelFormat.B8G8R8A8UIntNormalized, 96, CanvasAlphaMode.Premultiplied);
scaleEffect.Source = cb;
scaleEffect.Scale = new System.Numerics.Vector2((float)_width / 64, (float)_height / 64);
scaleEffect.InterpolationMode = CanvasImageInterpolation.Cubic;
scaleEffect.BorderMode = EffectBorderMode.Hard;
if (_graysclaleVideoFlag)
{
var grayScaleEffect = new GrayscaleEffect
{
BufferPrecision = CanvasBufferPrecision.Precision8UIntNormalized,
CacheOutput = false,
Source = inputBitmap
};
ds.DrawImage(grayScaleEffect);
}
else
{
ds.DrawImage(inputBitmap);
}
ds.DrawImage(scaleEffect, 0, 0, new Windows.Foundation.Rect {
Height = _height, Width = _width
}, _heatmapOpacity);
if (_generateDots)
{
for (int i = 0; i < _count; i++)
{
ds.FillCircle(yaw[i] * _width / 64, pitch[i] * _height / 64, _dotsRadius, _colors[i % 5]);
}
}
ds.FillRectangle(new Windows.Foundation.Rect {
Height = _height, Width = _width
}, solidColorBrush);
ds.Flush();
}
}
public async void ProcessFrame(ProcessVideoFrameContext context)
{
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var chromaKeyEffect = new ChromaKeyEffect
{
Color = ChromaColor,
Source = inputBitmap,
Tolerance = Tolerance,
Feather = true
};
//only load bg image if it hasnt already been loaded or if image is different
if (_backgroundBitmap == null || _lastUsedImageFilePath != _imageFileName)
{
_backgroundBitmap = await CanvasBitmap.LoadAsync(ds, new Uri($"ms-appdata:///Local/{ImageFileName}"));
}
_lastUsedImageFilePath = _imageFileName; //keep track of any incoming image changes
var compositeEffect = new CompositeEffect
{
Sources = { _backgroundBitmap, chromaKeyEffect }
};
ds.DrawImage(compositeEffect);
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Ignore))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var chroma = new ChromaKeyEffect
{
Color = ChromaColor,
Source = inputBitmap,
Tolerance = Tolerance
};
//only load bg image if it hasnt already been loaded or if image is different
if (_backgroundBitmap == null || _lastUsedImageFilePath != _imageFileName)
{
_backgroundBitmap = await CanvasBitmap.LoadAsync(ds, new Uri($"ms-appdata:///Local/{ImageFileName}"));
}
//keep track of any incoming image changes
_lastUsedImageFilePath = _imageFileName;
var compositeEffect = new CompositeEffect {
Mode = CanvasComposite.Add
};
compositeEffect.Sources.Add(_backgroundBitmap);
compositeEffect.Sources.Add(chroma);
ds.DrawImage(compositeEffect);
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputDirect3DSurface = context.InputFrame.Direct3DSurface;
float anyDpi = 96.0f;
var width = context.OutputFrame.Direct3DSurface.Description.Width;
var height = context.OutputFrame.Direct3DSurface.Description.Height;
// Part 1: Apply effect using Imaging SDK
CanvasRenderTarget renderTarget = new CanvasRenderTarget(canvasDevice, (float)width, (float)height, anyDpi);
if (inputDirect3DSurface != null && renderTarget != null)
{
if (m_direct3DSurfaceImageSource == null)
{
m_direct3DSurfaceImageSource = new Direct3DSurfaceImageSource();
}
m_direct3DSurfaceImageSource.Direct3DSurface = inputDirect3DSurface;
((IImageConsumer)m_grayScaleEffect).Source = m_direct3DSurfaceImageSource;
if (m_direct3DSurfaceRenderer == null)
{
m_direct3DSurfaceRenderer = new Direct3DSurfaceRenderer();
}
m_direct3DSurfaceRenderer.Direct3DSurface = renderTarget;
m_direct3DSurfaceRenderer.Source = m_grayScaleEffect;
var task = m_direct3DSurfaceRenderer.RenderAsync().AsTask();
task.Wait();
m_direct3DSurfaceRenderer.Direct3DSurface = null;
m_direct3DSurfaceImageSource.Direct3DSurface = null;
}
// Part 2: Apply another effect using Win2D
var size = context.OutputFrame.Direct3DSurface.Description.Width * context.OutputFrame.Direct3DSurface.Description.Height;
var colors = Enumerable.Repeat(Colors.Black, size).ToArray();
var inputBitmap = CanvasBitmap.CreateFromColors(canvasDevice, colors, width, height, anyDpi);
inputBitmap.CopyPixelsFromBitmap(renderTarget);
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
float dispX = (float)Math.Cos(time.TotalSeconds) * 75f;
float dispY = (float)Math.Sin(time.TotalSeconds) * 75f;
ds.Clear(Colors.Black);
var dispMap = new DisplacementMapEffect()
{
Source = inputBitmap,
XChannelSelect = EffectChannelSelect.Red,
YChannelSelect = EffectChannelSelect.Green,
Amount = 100f,
Displacement = new Transform2DEffect()
{
TransformMatrix = Matrix3x2.CreateTranslation(dispX, dispY),
Source = new BorderEffect()
{
ExtendX = CanvasEdgeBehavior.Mirror,
ExtendY = CanvasEdgeBehavior.Mirror,
Source = new TurbulenceEffect()
{
Octaves = 3
}
}
}
};
ds.DrawImage(dispMap, -25f, -25f);
}
((IDisposable)renderTarget).Dispose();
((IDisposable)inputBitmap).Dispose();
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
CanvasRenderTarget rt = new CanvasRenderTarget(canvasDevice, context.OutputFrame.Direct3DSurface.Description.Width, context.OutputFrame.Direct3DSurface.Description.Height, 96.0f);
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = rt.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
float dispX = (float)Math.Cos(time.TotalSeconds) * 75f;
float dispY = (float)Math.Sin(time.TotalSeconds) * 75f;
ds.Clear(Colors.Black);
//var posterizeEffect = new PosterizeEffect()
//{
// Source = input,
// BlueValueCount = 2,
// RedValueCount = 2,
// GreenValueCount = 2,
//};
var transformEffect = new Transform2DEffect()
{
Source = input,
TransformMatrix = new System.Numerics.Matrix3x2(-1, 0, 0, 1, rt.SizeInPixels.Width, 0),
};
var discreteTransferEffect = new DiscreteTransferEffect()
{
Source = transformEffect,
RedTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f },
GreenTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f },
BlueTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f },
AlphaDisable = true,
};
var dispMap = new ConvolveMatrixEffect()
{
KernelMatrix = new float[]
{
0, +1, 0,
+1, -4, +1,
0, +1, 0,
},
Source = discreteTransferEffect,
Divisor = 0.2f,
};
var modEffect = new ArithmeticCompositeEffect()
{
Source1 = dispMap,
Source2 = dispMap,
MultiplyAmount = 1,
};
//var finalPosterizeEffect = new PosterizeEffect()
//{
// Source = modEffect,
// BlueValueCount = 2,
// RedValueCount = 2,
// GreenValueCount = 2,
//};
//var dispMap = new EdgeDetectionEffect()
//{
// Source = greyScaleEffect,
// Mode = EdgeDetectionEffectMode.Sobel,
//};
Rect src = new Rect(rt.SizeInPixels.Width / 3, rt.SizeInPixels.Height / 3, rt.SizeInPixels.Width / 3, rt.SizeInPixels.Height / 3);
//ds.DrawImage(input, bottomLeft);
//ds.DrawImage(discreteTransferEffect, bottomRight, src);
//ds.DrawImage(dispMap, topLeft, src);
ds.DrawImage(modEffect, src, src);
}
int centerX = 0;
int centerY = 0;
int cubletWidth = 0;
int cubletHeight = 0;
int analysisAreaX = (int)rt.SizeInPixels.Width * 3 / 10;
int analysisWidth = (int)rt.SizeInPixels.Width * 4 / 10;
byte[] analysisHorzBytes = rt.GetPixelBytes(analysisAreaX, (int)rt.SizeInPixels.Height / 2, analysisWidth, 1);
int analysisAreaY = (int)rt.SizeInPixels.Height * 3 / 10;
int analysisHeight = (int)rt.SizeInPixels.Height * 4 / 10;
byte[] analysisVertBytes = rt.GetPixelBytes((int)rt.SizeInPixels.Width / 2, analysisAreaY, 1, analysisHeight);
int foundLeft = 0;
for (int i = 0; i < analysisWidth / 2; i++)
{
byte b = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 0];
byte g = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 1];
byte r = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundLeft == 0)
{
foundLeft = i;
}
}
int foundRight = 0;
for (int i = 0; i < analysisWidth / 2; i++)
{
byte r = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 0];
byte g = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 1];
byte b = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundRight == 0)
{
foundRight = i;
}
}
int foundTop = 0;
for (int i = 0; i < analysisHeight / 2; i++)
{
byte r = analysisVertBytes[4 * (analysisHeight / 2 - i) + 0];
byte g = analysisVertBytes[4 * (analysisHeight / 2 - i) + 1];
byte b = analysisVertBytes[4 * (analysisHeight / 2 - i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundTop == 0)
{
foundTop = i;
}
}
int foundBottom = 0;
for (int i = 0; i < analysisHeight / 2; i++)
{
byte r = analysisVertBytes[4 * (analysisHeight / 2 + i) + 0];
byte g = analysisVertBytes[4 * (analysisHeight / 2 + i) + 1];
byte b = analysisVertBytes[4 * (analysisHeight / 2 + i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundBottom == 0)
{
foundBottom = i;
}
}
centerX = (-foundLeft + foundRight) / 2 + (int)rt.SizeInPixels.Width / 2;
centerY = (-foundTop + foundBottom) / 2 + (int)rt.SizeInPixels.Height / 2;
cubletWidth = (int)((foundLeft + foundRight) * 1.2f);
cubletHeight = (int)((foundTop + foundBottom) * 1.2f);
// No 2d arrays in WinRT components? Boo.
// "Error C1113 #using failed on 'rubikscuberecognitionwinrt.winmd' RubiksCubeRecognitionLib
Vector2[] cubletCenters = new Vector2[3 * 3];
cubletCenters[1 * 3 + 1] = new Vector2(centerX, centerY);
cubletCenters[1 * 3 + 0] = new Vector2(centerX, centerY - cubletHeight);
cubletCenters[1 * 3 + 2] = new Vector2(centerX, centerY + cubletHeight);
cubletCenters[0 * 3 + 1] = new Vector2(centerX - cubletWidth, centerY);
cubletCenters[0 * 3 + 0] = new Vector2(centerX - cubletWidth, centerY - cubletHeight);
cubletCenters[0 * 3 + 2] = new Vector2(centerX - cubletWidth, centerY + cubletHeight);
cubletCenters[2 * 3 + 1] = new Vector2(centerX + cubletWidth, centerY);
cubletCenters[2 * 3 + 0] = new Vector2(centerX + cubletWidth, centerY - cubletHeight);
cubletCenters[2 * 3 + 2] = new Vector2(centerX + cubletWidth, centerY + cubletHeight);
cubeletColorsMutex.WaitOne();
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
var transformEffect = new Transform2DEffect()
{
Source = input,
TransformMatrix = new System.Numerics.Matrix3x2(-1, 0, 0, 1, output.SizeInPixels.Width, 0),
};
Rect src = new Rect(0, 0, output.SizeInPixels.Width, output.SizeInPixels.Height);
ds.DrawImage(transformEffect, src, src);
// Draw a crosshair on the screen
ds.FillRectangle(new Rect(output.SizeInPixels.Width / 2 - 3, output.SizeInPixels.Height / 4, 6, output.SizeInPixels.Height / 2), Colors.Gray);
ds.FillRectangle(new Rect(output.SizeInPixels.Width / 4, output.SizeInPixels.Height / 2 - 3, output.SizeInPixels.Width / 2, 6), Colors.Gray);
if (true)
{
for (int x = 0; x < 3; x++)
{
for (int y = 0; y < 3; y++)
{
int sampleWidth = 2;
byte[] cubletBytes = input.GetPixelBytes((int)cubletCenters[(2 - x) * 3 + y].X - sampleWidth / 2, (int)cubletCenters[(2 - x) * 3 + y].Y - sampleWidth / 2, sampleWidth, sampleWidth);
int totalR = 0; int totalG = 0; int totalB = 0;
for (int i = 0; i < sampleWidth * sampleWidth; i++)
{
totalB += cubletBytes[4 * i + 0];
totalG += cubletBytes[4 * i + 1];
totalR += cubletBytes[4 * i + 2];
}
cubletColors[x * 3 + y] = Color.FromArgb(255, (byte)(totalR / (sampleWidth * sampleWidth)), (byte)(totalG / (sampleWidth * sampleWidth)), (byte)(totalB / (sampleWidth * sampleWidth)));
}
}
for (int x = 0; x < 3; x++)
{
for (int y = 0; y < 3; y++)
{
ds.FillRectangle((float)cubletCenters[x * 3 + y].X - 12, (float)cubletCenters[x * 3 + y].Y - 12, 24, 24, Colors.Black);
ds.FillRectangle((float)cubletCenters[x * 3 + y].X - 10, (float)cubletCenters[x * 3 + y].Y - 10, 20, 20, cubletColors[x * 3 + y]);
}
}
}
}
cubeletColorsMutex.ReleaseMutex();
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
//using (var input = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
//using (var output = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (var ds = output.CreateDrawingSession())
//{
// var time = context.InputFrame.RelativeTime ?? new TimeSpan();
// ds.Clear(Colors.Black);
// for (uint i = 0; i < _numColumns; i++)
// {
// for (uint j = 0; j < _numRows; j++)
// {
// _crops[i, j].Source = input;
// float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
// float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
// Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile);
// _transforms[i, j].TransformMatrix =
// Matrix3x2.CreateRotation(rotation, centerPoint) *
// Matrix3x2.CreateScale(scale, centerPoint);
// ds.DrawImage(_transforms[i, j]);
// }
// }
//}
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var time = context.InputFrame.RelativeTime ?? new TimeSpan();
ds.Clear(Colors.Black);
for (uint i = 0; i < _numColumns; i++)
{
for (uint j = 0; j < _numRows; j++)
{
_crops[i, j].Source = inputBitmap;
float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile);
_transforms[i, j].TransformMatrix =
Matrix3x2.CreateRotation(rotation, centerPoint) *
Matrix3x2.CreateScale(scale, centerPoint);
ds.DrawImage(_transforms[i, j]);
}
}
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var time = context.InputFrame.RelativeTime ?? new TimeSpan();
ds.Clear(Colors.Black);
for (uint i = 0; i < _numColumns; i++)
{
for (uint j = 0; j < _numRows; j++)
{
_crops[i, j].Source = inputBitmap;
float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile);
_transforms[i, j].TransformMatrix =
Matrix3x2.CreateRotation(rotation, centerPoint) *
Matrix3x2.CreateScale(scale, centerPoint);
ds.DrawImage(_transforms[i, j]);
}
}
}
}
}
}