public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
ds.Clear(Colors.Black);
for (uint i = 0; i < numColumns; i++)
{
for (uint j = 0; j < numRows; j++)
{
crops[i, j].Source = input;
float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
Vector2 centerPoint = new Vector2((i + 0.5f) * pixelsPerTile, (j + 0.5f) * pixelsPerTile);
transforms[i, j].TransformMatrix =
Matrix3x2.CreateRotation(rotation, centerPoint) *
Matrix3x2.CreateScale(scale, centerPoint);
ds.DrawImage(transforms[i, j]);
}
}
}
}
public void ProcessFrameGPU(ProcessVideoFrameContext context)
{
var inputSurface = context.InputFrame.Direct3DSurface;
var outputSurface = context.OutputFrame.Direct3DSurface;
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(m_canvasDevice, inputSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(m_canvasDevice, outputSurface))
using (var drawingSession = renderTarget.CreateDrawingSession())
{
var normalize = new Matrix5x4()
{
M11 = 1f, M22 = 1f, M33 = 1f, M44 = 1f, M51 = -m_mean.X, M52 = -m_mean.Y, M53 = -m_mean.Z
};
var normalize2 = new Matrix5x4()
{
M11 = 1 / m_std.X, M22 = 1 / m_std.Y, M33 = 1 / m_std.Z, M44 = 1.0f
};
// https://microsoft.github.io/Win2D/html/T_Microsoft_Graphics_Canvas_Effects_ColorMatrixEffect.htm
var PreprocessTransfrom = new ColorMatrixEffect()
{
ColorMatrix = normalize2,
Source = new ColorMatrixEffect()
{
ColorMatrix = normalize,
Source = inputBitmap
}
};
drawingSession.DrawImage(PreprocessTransfrom);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
ds.Clear(Colors.Black);
for (uint i = 0; i < numColumns; i++)
{
for (uint j = 0; j < numRows; j++)
{
crops[i, j].Source = input;
float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
Vector2 centerPoint = new Vector2((i + 0.5f) * pixelsPerTile, (j + 0.5f) * pixelsPerTile);
transforms[i, j].TransformMatrix =
Matrix3x2.CreateRotation(rotation, centerPoint) *
Matrix3x2.CreateScale(scale, centerPoint);
ds.DrawImage(transforms[i, j]);
}
}
}
}
/// <summary>
/// Used for applying our video effect to a single frame.
/// </summary>
/// <param name="context"></param>
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputFrameBitmap = context.InputFrame.SoftwareBitmap;
// Create intermediate buffer for holding the frame pixel data.
var frameSize = inputFrameBitmap.PixelWidth * inputFrameBitmap.PixelHeight * 4;
var frameBuffer = new Buffer((uint)frameSize);
// Copy bitmap data from the input frame.
inputFrameBitmap.CopyToBuffer(frameBuffer);
// Iterate through all pixels in the frame.
var framePixels = frameBuffer.ToArray();
for (int i = 0; i < frameSize; i += 4)
{
// Calculate the luminance based on the RGB values - this way we can convert it to grayscale.
var bValue = framePixels[i];
var gValue = framePixels[i + 1];
var rValue = framePixels[i + 2];
var luminance = ((rValue / 255.0f) * 0.2126f) +
((gValue / 255.0f) * 0.7152f) +
((bValue / 255.0f) * 0.0722f);
// Set the pixel data to the calculated grayscale values.
framePixels[i] = framePixels[i + 1] = framePixels[i + 2] = (byte)(luminance * 255.0f);
}
// Copy the modified frame data to the output frame.
context.OutputFrame.SoftwareBitmap.CopyFromBuffer(framePixels.AsBuffer());
}
/// <summary>
/// Used for applying our video effect to a single frame.
/// </summary>
/// <param name="context"></param>
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputFrameBitmap = context.InputFrame.SoftwareBitmap;
// Create intermediate buffer for holding the frame pixel data.
var frameSize = inputFrameBitmap.PixelWidth * inputFrameBitmap.PixelHeight * 4;
var frameBuffer = new Buffer((uint)frameSize);
// Copy bitmap data from the input frame.
inputFrameBitmap.CopyToBuffer(frameBuffer);
// Iterate through all pixels in the frame.
var framePixels = frameBuffer.ToArray();
for (int i = 0; i < frameSize; i += 4)
{
// Calculate the luminance based on the RGB values - this way we can convert it to grayscale.
var bValue = framePixels[i];
var gValue = framePixels[i + 1];
var rValue = framePixels[i + 2];
var luminance = ((rValue / 255.0f) * 0.2126f) +
((gValue / 255.0f) * 0.7152f) +
((bValue / 255.0f) * 0.0722f);
// Set the pixel data to the calculated grayscale values.
framePixels[i] = framePixels[i + 1] = framePixels[i + 2] = (byte)(luminance * 255.0f);
}
// Copy the modified frame data to the output frame.
context.OutputFrame.SoftwareBitmap.CopyFromBuffer(framePixels.AsBuffer());
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputSurface = context.InputFrame.Direct3DSurface;
var outputSurface = context.OutputFrame.Direct3DSurface;
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, inputSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, outputSurface))
using (var ds = renderTarget.CreateDrawingSession())
using (var brush = new CanvasImageBrush(canvasDevice, inputBitmap))
using (var textCommandList = new CanvasCommandList(canvasDevice))
{
using (var clds = textCommandList.CreateDrawingSession())
{
clds.DrawText(
"Win2D\nMediaClip",
(float)inputBitmap.Size.Width / 2,
(float)inputBitmap.Size.Height / 2,
brush,
new CanvasTextFormat()
{
FontSize = (float)inputBitmap.Size.Width / 5,
FontWeight = new FontWeight()
{
Weight = 999
},
HorizontalAlignment = CanvasHorizontalAlignment.Center,
VerticalAlignment = CanvasVerticalAlignment.Center
});
}
var background = new GaussianBlurEffect()
{
BlurAmount = 10,
BorderMode = EffectBorderMode.Hard,
Source = new BrightnessEffect()
{
BlackPoint = new Vector2(0.5f, 0.7f),
Source = new SaturationEffect()
{
Saturation = 0,
Source = inputBitmap
}
}
};
var shadow = new ShadowEffect()
{
Source = textCommandList,
BlurAmount = 10
};
var composite = new CompositeEffect()
{
Sources = { background, shadow, textCommandList }
};
ds.DrawImage(composite);
}
}
// ** Methods ** //
// This is run for every video frame passed in the media pipeline (MediaPlayer, MediaCapture, etc)
public void ProcessFrame(ProcessVideoFrameContext context)
{
videoFrameToProcess = VideoFrame.CreateWithDirect3D11Surface(context.InputFrame.Direct3DSurface);
// ********** Draw Bounding Boxes with Win2D ********** //
// Use Direct3DSurface if using GPU memory
if (context.InputFrame.Direct3DSurface != null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
foreach (var box in filteredBoxes)
{
var x = (uint)Math.Max(box.X, 0);
var y = (uint)Math.Max(box.Y, 0);
var w = (uint)Math.Min(renderTarget.Bounds.Width - x, box.Width);
var h = (uint)Math.Min(renderTarget.Bounds.Height - y, box.Height);
// Draw the Text 10px above the top of the bounding box
ds.DrawText(box.Label, x, y - 10, Colors.Yellow);
ds.DrawRectangle(new Rect(x, y, w, h), new CanvasSolidColorBrush(canvasDevice, Colors.Yellow), 2f);
}
}
return;
}
// Use SoftwareBitmap if using CPU memory
if (context.InputFrame.SoftwareBitmap != null)
{
// InputFrame's pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(canvasDevice, inputFrameBytes, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(canvasDevice, context.OutputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, (float)context.OutputFrame.SoftwareBitmap.DpiX, context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(), CanvasAlphaMode.Premultiplied))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
foreach (var box in filteredBoxes)
{
var x = (uint)Math.Max(box.X, 0);
var y = (uint)Math.Max(box.Y, 0);
var w = (uint)Math.Min(context.OutputFrame.SoftwareBitmap.PixelWidth - x, box.Width);
var h = (uint)Math.Min(context.OutputFrame.SoftwareBitmap.PixelHeight - y, box.Height);
// Draw the Text 10px above the top of the bounding box
ds.DrawText(box.Label, x, y - 10, Colors.Yellow);
ds.DrawRectangle(new Rect(x, y, w, h), new CanvasSolidColorBrush(canvasDevice, Colors.Yellow), 2f);
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
// If memory type is CPU, the frame is in InputFrame.SoftwareBitmap.
// For GPU, the frame is in InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var sepia = new SepiaEffect
{
Source = inputBitmap,
Intensity = this.Intensity
};
ds.DrawImage(sepia);
}
return;
}
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var sepia = new SepiaEffect
{
Source = inputBitmap,
Intensity = this.Intensity
};
ds.DrawImage(sepia);
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputSurface = context.InputFrame.Direct3DSurface;
var outputSurface = context.OutputFrame.Direct3DSurface;
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, inputSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, outputSurface))
using (var ds = renderTarget.CreateDrawingSession())
using (var brush = new CanvasImageBrush(canvasDevice, inputBitmap))
using (var textCommandList = new CanvasCommandList(canvasDevice))
{
using (var clds = textCommandList.CreateDrawingSession())
{
clds.DrawText(
"Win2D\nMediaClip",
(float)inputBitmap.Size.Width / 2,
(float)inputBitmap.Size.Height / 2,
brush,
new CanvasTextFormat()
{
FontSize = (float)inputBitmap.Size.Width / 5,
FontWeight = new FontWeight() { Weight = 999 },
HorizontalAlignment = CanvasHorizontalAlignment.Center,
VerticalAlignment = CanvasVerticalAlignment.Center
});
}
var background = new GaussianBlurEffect()
{
BlurAmount = 10,
BorderMode = EffectBorderMode.Hard,
Source = new BrightnessEffect()
{
BlackPoint = new Vector2(0.5f, 0.7f),
Source = new SaturationEffect()
{
Saturation = 0,
Source = inputBitmap
}
}
};
var shadow = new ShadowEffect()
{
Source = textCommandList,
BlurAmount = 10
};
var composite = new CompositeEffect()
{
Sources = { background, shadow, textCommandList }
};
ds.DrawImage(composite);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.Transform = MediaExtension.CreateTransform(inputBitmap);
ds.DrawImage(inputBitmap);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
bool skipMaskPred = false;
if (context.InputFrame.IsDiscontinuous)
{
streamStartDelta = TimeSpan.FromTicks(DateTime.Now.Ticks) - context.InputFrame.SystemRelativeTime.Value;
}
else
{
if ((TimeSpan.FromTicks(DateTime.Now.Ticks) - context.InputFrame.SystemRelativeTime.Value - streamStartDelta) > TimeSpan.FromMilliseconds(maxDelay))
{
skipMaskPred = true;
}
}
if (!skipMaskPred)
{
frameCount++;
features["0"] = context.InputFrame;
var resTask = _learningModel.EvaluateFeaturesAsync(features, string.Empty).AsTask();
var startTime = DateTime.Now.Ticks;
resTask.Wait();
Debug.WriteLine("delta {0}", TimeSpan.FromTicks(DateTime.Now.Ticks - startTime));
}
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasBitmap inputMask = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, output.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
ds.Clear(Colors.Green);
var addAlpha = new ColorMatrixEffect()
{
Source = inputMask,
ColorMatrix = RToAlpha
};
var resize = new ScaleEffect()
{
Source = addAlpha,
Scale = new Vector2(((float)inputBitmap.SizeInPixels.Width / inputMask.SizeInPixels.Width), ((float)inputBitmap.SizeInPixels.Height / inputMask.SizeInPixels.Height))
};
var blend = new AlphaMaskEffect()
{
Source = inputBitmap,
AlphaMask = resize
};
ds.DrawImage(blend);
ds.DrawText(String.Format("FPS: {0:f1}", currentFPS), fpsLabelOffset, fpsLabelColor);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
if (context.InputFrame.SoftwareBitmap != null)
{
ProcessFrameCPU(context);
}
else if (context.InputFrame.Direct3DSurface != null)
{
ProcessFrameGPU(context);
}
}
//private ICanvasImage[] BackgroundImages { get; set; } = default(ICanvasImage[]);
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
using (var chromaKeyEffect = MediaExtension.CreateChromaKeyEffect(inputBitmap))
{
ds.Blend = CanvasBlend.Copy;
ds.DrawImage(chromaKeyEffect);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
var saturation = new SaturationEffect()
{
Source = inputBitmap,
Saturation = this.Saturation
};
ds.DrawImage(saturation);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(MediaExtension.Device, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(MediaExtension.Device, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
if (MediaExtension.ActiveBackgroundSet != null &&
MediaExtension.BackgroundSetCache != null &&
MediaExtension.BackgroundSetCache.TryGetValue(
key: MediaExtension.ActiveBackgroundSet, value: out var set) &&
set.IsCompleted &&
set.Result.Length > 0)
{
try
{
set.Wait();
}
catch (AggregateException exception)
{
// .NET async tasks wrap all errors in an AggregateException.
// We unpack this so Win2D can directly see any lost device errors.
exception.Handle(ex => { throw ex; });
}
var framesPerMillisecond = MediaExtension.ActiveBackgroundFramesPerSecond / 1000;
var durationInMilliseconds = set.Result.Length / framesPerMillisecond;
var offsetInDuration = context.InputFrame.RelativeTime?.TotalMilliseconds % durationInMilliseconds / durationInMilliseconds ?? 0;
var offsetInFrames = Convert.ToInt32(set.Result.Length * offsetInDuration);
// Does someone want to fix my algorithm so this isn't necessary?
var offsetInIndex = offsetInFrames <= 0 ? 0
: offsetInFrames >= set.Result.Length ? set.Result.Length - 1
: offsetInFrames;
var canvas = set.Result[offsetInIndex];
var scale = inputBitmap.Bounds.Width / canvas.Bounds.Width;
var bounds = canvas.Bounds;
bounds.Width = bounds.Width * scale;
bounds.Height = bounds.Height * scale;
ds.DrawImage(canvas, bounds);
}
else
{
ds.FillRectangle(inputBitmap.Bounds, Colors.Black);
}
ds.DrawImage(inputBitmap);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
var saturation = new SaturationEffect()
{
Source = inputBitmap,
Saturation = this.Saturation
};
ds.DrawImage(saturation);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
//Getingt pixel data
var inputFrameBitmap = context.InputFrame.SoftwareBitmap;
var frameSize = inputFrameBitmap.PixelWidth * inputFrameBitmap.PixelHeight * 4;
var frameBuffer = new Buffer((uint)frameSize);
context.InputFrame.SoftwareBitmap.CopyToBuffer(frameBuffer);
var framePixels = frameBuffer.ToArray();
Debug.WriteLine($"Touching 25th red pixel {framePixels[100]}");
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(_device, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget target = CanvasRenderTarget.CreateFromDirect3D11Surface(_device, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession session = target.CreateDrawingSession())
{
byte[] bytes = input.GetPixelBytes();
IntPtr ptr = Marshal.AllocHGlobal(bytes.Length);
Marshal.Copy(bytes, 0, ptr, bytes.Length);
Marshal.FreeHGlobal(ptr);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (var output = CanvasRenderTarget.CreateFromDirect3D11Surface(device, context.OutputFrame.Direct3DSurface))
using (var input = CanvasRenderTarget.CreateFromDirect3D11Surface(device, context.InputFrame.Direct3DSurface))
{
var args = new VideoEffectHandlerArgs()
{
ID = id,
Device = device,
InputFrame = input,
OutputFrame = output,
Properties = properties
};
VideoEffectManager.ProcessFrame(args);
}
}
// </SnippetBlurAmountWin2D>
// <SnippetProcessFrameWin2D>
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
var gaussianBlurEffect = new GaussianBlurEffect
{
Source = inputBitmap,
BlurAmount = (float)BlurAmount,
Optimization = EffectOptimization.Speed
};
ds.DrawImage(gaussianBlurEffect);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
ColorMatrixEffect colorMatrixEffect = new ColorMatrixEffect
{
Source = inputBitmap
};
colorMatrixEffect.ColorMatrix = ColorMatrix;
ds.DrawImage(colorMatrixEffect);
}
}
public unsafe void ProcessFrame(ProcessVideoFrameContext context)
{
frameCount++;
Debug.WriteLine("Frame count: " + frameCount);
using (BitmapBuffer buffer = context.InputFrame.SoftwareBitmap.LockBuffer(BitmapBufferAccessMode.Read))
using (BitmapBuffer targetBuffer = context.OutputFrame.SoftwareBitmap.LockBuffer(BitmapBufferAccessMode.Write))
{
using (var reference = buffer.CreateReference())
using (var targetReference = targetBuffer.CreateReference())
{
byte *dataInBytes;
uint capacity;
((IMemoryBufferByteAccess)reference).GetBuffer(out dataInBytes, out capacity);
byte *targetDataInBytes;
uint targetCapacity;
((IMemoryBufferByteAccess)targetReference).GetBuffer(out targetDataInBytes, out targetCapacity);
var fadeValue = FadeValue;
// Fill-in the BGRA plane
BitmapPlaneDescription bufferLayout = buffer.GetPlaneDescription(0);
for (int i = 0; i < bufferLayout.Height; i++)
{
for (int j = 0; j < bufferLayout.Width; j++)
{
byte value = (byte)((float)j / bufferLayout.Width * 255);
int bytesPerPixel = 4;
if (encodingProperties.Subtype != "ARGB32")
{
// If you support other encodings, adjust index into the buffer accordingly
}
int idx = bufferLayout.StartIndex + bufferLayout.Stride * i + bytesPerPixel * j;
targetDataInBytes[idx + 0] = (byte)(fadeValue * (float)dataInBytes[idx + 0]);
//targetDataInBytes[idx + 1] = (byte)(fadeValue * (float)dataInBytes[idx + 1]);
//targetDataInBytes[idx + 2] = (byte)(fadeValue * (float)dataInBytes[idx + 2]);
//targetDataInBytes[idx + 3] = dataInBytes[idx + 3];
}
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
ds.Clear(Colors.Black);
var mirrorEffect = new Transform2DEffect()
{
Source = input,
TransformMatrix = new Matrix3x2(-1, 0, 0, 1, output.SizeInPixels.Width, 0),
};
ds.DrawImage(mirrorEffect);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
ds.Clear(Colors.Black);
var mirrorEffect = new Transform2DEffect()
{
Source = input,
TransformMatrix = new Matrix3x2(-1, 0, 0, 1, output.SizeInPixels.Width, 0),
};
ds.DrawImage(mirrorEffect);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
var gaussianBlurEffect = new GaussianBlurEffect
{
Source = inputBitmap,
BlurAmount = (float)BlurAmount,
Optimization = EffectOptimization.Speed
};
ds.DrawImage(gaussianBlurEffect);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
{
var mills = 2000.0 - (context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value.TotalMilliseconds - 2000.0 : 0.0);
var k = mills > 0 ? (float)mills / 2000.0f : 0.0f;
var gaussianBlurEffect = new GaussianBlurEffect
{
Source = inputBitmap,
BlurAmount = 30 * k,
Optimization = EffectOptimization.Speed
};
ds.DrawImage(gaussianBlurEffect);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputSurface = context.InputFrame.Direct3DSurface;
var outputSurface = context.OutputFrame.Direct3DSurface;
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, inputSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, outputSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
uint ww = inputBitmap.SizeInPixels.Width;
uint hh = inputBitmap.SizeInPixels.Height;
Rect rx = new Rect(0, 0, ww, hh);
ds.DrawRectangle(rx, Colors.White);
uint ww2 = ww - 2;
uint hh2 = hh - 2;
Rect rxInner = new Rect(1, 1, ww2, hh2);
ds.DrawRectangle(rx, Colors.Black);
}
}
public unsafe void ProcessFrameCPU(ProcessVideoFrameContext context)
{
using (BitmapBuffer buffer = context.InputFrame.SoftwareBitmap.LockBuffer(BitmapBufferAccessMode.Read))
using (BitmapBuffer targetBuffer = context.OutputFrame.SoftwareBitmap.LockBuffer(BitmapBufferAccessMode.Write))
{
using (var reference = buffer.CreateReference())
using (var targetReference = targetBuffer.CreateReference())
{
byte *dataInBytes;
uint capacity;
((IMemoryBufferByteAccess)reference).GetBuffer(out dataInBytes, out capacity);
byte *targetDataInBytes;
uint targetCapacity;
((IMemoryBufferByteAccess)targetReference).GetBuffer(out targetDataInBytes, out targetCapacity);
// Fill-in the BGRA plane
BitmapPlaneDescription bufferLayout = buffer.GetPlaneDescription(0);
var bytesPerPixel = 4;
for (int i = 0; i < bufferLayout.Height; i++)
{
for (int j = 0; j < bufferLayout.Width; j++)
{
if (m_encodingProperties.Subtype != "ARGB32")
{
// If you support other encodings, adjust index into the buffer accordingly
}
int idx = bufferLayout.StartIndex + bufferLayout.Stride * i + bytesPerPixel * j;
targetDataInBytes[idx + 0] = (byte)((dataInBytes[idx + 0] - m_mean.X) / m_std.X);
targetDataInBytes[idx + 1] = (byte)((dataInBytes[idx + 1] - m_mean.Y) / m_std.Y);
targetDataInBytes[idx + 2] = (byte)((dataInBytes[idx + 2] - m_mean.Z) / m_std.Z);
targetDataInBytes[idx + 3] = dataInBytes[idx + 3];
}
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
float dispX = (float)Math.Cos(time.TotalSeconds) * 75f;
float dispY = (float)Math.Sin(time.TotalSeconds) * 75f;
ds.Clear(Colors.Black);
var dispMap = new DisplacementMapEffect()
{
Source = input,
XChannelSelect = EffectChannelSelect.Red,
YChannelSelect = EffectChannelSelect.Green,
Amount = 100f,
Displacement = new Transform2DEffect()
{
TransformMatrix = Matrix3x2.CreateTranslation(dispX, dispY),
Source = new BorderEffect()
{
ExtendX = CanvasEdgeBehavior.Mirror,
ExtendY = CanvasEdgeBehavior.Mirror,
Source = new TurbulenceEffect()
{
Octaves = 3
}
}
}
};
ds.DrawImage(dispMap, -25f, -25f);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
float dispX = (float)Math.Cos(time.TotalSeconds) * 75f;
float dispY = (float)Math.Sin(time.TotalSeconds) * 75f;
ds.Clear(Colors.Black);
var dispMap = new DisplacementMapEffect()
{
Source = input,
XChannelSelect = EffectChannelSelect.Red,
YChannelSelect = EffectChannelSelect.Green,
Amount = 100f,
Displacement = new Transform2DEffect()
{
TransformMatrix = Matrix3x2.CreateTranslation(dispX, dispY),
Source = new BorderEffect()
{
ExtendX = CanvasEdgeBehavior.Mirror,
ExtendY = CanvasEdgeBehavior.Mirror,
Source = new TurbulenceEffect()
{
Octaves = 3
}
}
}
};
ds.DrawImage(dispMap, -25f, -25f);
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
CanvasRenderTarget rt = new CanvasRenderTarget(canvasDevice, context.OutputFrame.Direct3DSurface.Description.Width, context.OutputFrame.Direct3DSurface.Description.Height, 96.0f);
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = rt.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
float dispX = (float)Math.Cos(time.TotalSeconds) * 75f;
float dispY = (float)Math.Sin(time.TotalSeconds) * 75f;
ds.Clear(Colors.Black);
//var posterizeEffect = new PosterizeEffect()
//{
// Source = input,
// BlueValueCount = 2,
// RedValueCount = 2,
// GreenValueCount = 2,
//};
var transformEffect = new Transform2DEffect()
{
Source = input,
TransformMatrix = new System.Numerics.Matrix3x2(-1, 0, 0, 1, rt.SizeInPixels.Width, 0),
};
var discreteTransferEffect = new DiscreteTransferEffect()
{
Source = transformEffect,
RedTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f },
GreenTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f },
BlueTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f },
AlphaDisable = true,
};
var dispMap = new ConvolveMatrixEffect()
{
KernelMatrix = new float[]
{
0, +1, 0,
+1, -4, +1,
0, +1, 0,
},
Source = discreteTransferEffect,
Divisor = 0.2f,
};
var modEffect = new ArithmeticCompositeEffect()
{
Source1 = dispMap,
Source2 = dispMap,
MultiplyAmount = 1,
};
//var finalPosterizeEffect = new PosterizeEffect()
//{
// Source = modEffect,
// BlueValueCount = 2,
// RedValueCount = 2,
// GreenValueCount = 2,
//};
//var dispMap = new EdgeDetectionEffect()
//{
// Source = greyScaleEffect,
// Mode = EdgeDetectionEffectMode.Sobel,
//};
Rect src = new Rect(rt.SizeInPixels.Width / 3, rt.SizeInPixels.Height / 3, rt.SizeInPixels.Width / 3, rt.SizeInPixels.Height / 3);
//ds.DrawImage(input, bottomLeft);
//ds.DrawImage(discreteTransferEffect, bottomRight, src);
//ds.DrawImage(dispMap, topLeft, src);
ds.DrawImage(modEffect, src, src);
}
int centerX = 0;
int centerY = 0;
int cubletWidth = 0;
int cubletHeight = 0;
int analysisAreaX = (int)rt.SizeInPixels.Width * 3 / 10;
int analysisWidth = (int)rt.SizeInPixels.Width * 4 / 10;
byte[] analysisHorzBytes = rt.GetPixelBytes(analysisAreaX, (int)rt.SizeInPixels.Height / 2, analysisWidth, 1);
int analysisAreaY = (int)rt.SizeInPixels.Height * 3 / 10;
int analysisHeight = (int)rt.SizeInPixels.Height * 4 / 10;
byte[] analysisVertBytes = rt.GetPixelBytes((int)rt.SizeInPixels.Width / 2, analysisAreaY, 1, analysisHeight);
int foundLeft = 0;
for (int i = 0; i < analysisWidth / 2; i++)
{
byte b = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 0];
byte g = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 1];
byte r = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundLeft == 0)
{
foundLeft = i;
}
}
int foundRight = 0;
for (int i = 0; i < analysisWidth / 2; i++)
{
byte r = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 0];
byte g = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 1];
byte b = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundRight == 0)
{
foundRight = i;
}
}
int foundTop = 0;
for (int i = 0; i < analysisHeight / 2; i++)
{
byte r = analysisVertBytes[4 * (analysisHeight / 2 - i) + 0];
byte g = analysisVertBytes[4 * (analysisHeight / 2 - i) + 1];
byte b = analysisVertBytes[4 * (analysisHeight / 2 - i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundTop == 0)
{
foundTop = i;
}
}
int foundBottom = 0;
for (int i = 0; i < analysisHeight / 2; i++)
{
byte r = analysisVertBytes[4 * (analysisHeight / 2 + i) + 0];
byte g = analysisVertBytes[4 * (analysisHeight / 2 + i) + 1];
byte b = analysisVertBytes[4 * (analysisHeight / 2 + i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundBottom == 0)
{
foundBottom = i;
}
}
centerX = (-foundLeft + foundRight) / 2 + (int)rt.SizeInPixels.Width / 2;
centerY = (-foundTop + foundBottom) / 2 + (int)rt.SizeInPixels.Height / 2;
cubletWidth = (int)((foundLeft + foundRight) * 1.2f);
cubletHeight = (int)((foundTop + foundBottom) * 1.2f);
// No 2d arrays in WinRT components? Boo.
// "Error C1113 #using failed on 'rubikscuberecognitionwinrt.winmd' RubiksCubeRecognitionLib
Vector2[] cubletCenters = new Vector2[3 * 3];
cubletCenters[1 * 3 + 1] = new Vector2(centerX, centerY);
cubletCenters[1 * 3 + 0] = new Vector2(centerX, centerY - cubletHeight);
cubletCenters[1 * 3 + 2] = new Vector2(centerX, centerY + cubletHeight);
cubletCenters[0 * 3 + 1] = new Vector2(centerX - cubletWidth, centerY);
cubletCenters[0 * 3 + 0] = new Vector2(centerX - cubletWidth, centerY - cubletHeight);
cubletCenters[0 * 3 + 2] = new Vector2(centerX - cubletWidth, centerY + cubletHeight);
cubletCenters[2 * 3 + 1] = new Vector2(centerX + cubletWidth, centerY);
cubletCenters[2 * 3 + 0] = new Vector2(centerX + cubletWidth, centerY - cubletHeight);
cubletCenters[2 * 3 + 2] = new Vector2(centerX + cubletWidth, centerY + cubletHeight);
cubeletColorsMutex.WaitOne();
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
var transformEffect = new Transform2DEffect()
{
Source = input,
TransformMatrix = new System.Numerics.Matrix3x2(-1, 0, 0, 1, output.SizeInPixels.Width, 0),
};
Rect src = new Rect(0, 0, output.SizeInPixels.Width, output.SizeInPixels.Height);
ds.DrawImage(transformEffect, src, src);
// Draw a crosshair on the screen
ds.FillRectangle(new Rect(output.SizeInPixels.Width / 2 - 3, output.SizeInPixels.Height / 4, 6, output.SizeInPixels.Height / 2), Colors.Gray);
ds.FillRectangle(new Rect(output.SizeInPixels.Width / 4, output.SizeInPixels.Height / 2 - 3, output.SizeInPixels.Width / 2, 6), Colors.Gray);
if (true)
{
for (int x = 0; x < 3; x++)
{
for (int y = 0; y < 3; y++)
{
int sampleWidth = 2;
byte[] cubletBytes = input.GetPixelBytes((int)cubletCenters[(2 - x) * 3 + y].X - sampleWidth / 2, (int)cubletCenters[(2 - x) * 3 + y].Y - sampleWidth / 2, sampleWidth, sampleWidth);
int totalR = 0; int totalG = 0; int totalB = 0;
for (int i = 0; i < sampleWidth * sampleWidth; i++)
{
totalB += cubletBytes[4 * i + 0];
totalG += cubletBytes[4 * i + 1];
totalR += cubletBytes[4 * i + 2];
}
cubletColors[x * 3 + y] = Color.FromArgb(255, (byte)(totalR / (sampleWidth * sampleWidth)), (byte)(totalG / (sampleWidth * sampleWidth)), (byte)(totalB / (sampleWidth * sampleWidth)));
}
}
for (int x = 0; x < 3; x++)
{
for (int y = 0; y < 3; y++)
{
ds.FillRectangle((float)cubletCenters[x * 3 + y].X - 12, (float)cubletCenters[x * 3 + y].Y - 12, 24, 24, Colors.Black);
ds.FillRectangle((float)cubletCenters[x * 3 + y].X - 10, (float)cubletCenters[x * 3 + y].Y - 10, 20, 20, cubletColors[x * 3 + y]);
}
}
}
}
cubeletColorsMutex.ReleaseMutex();
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputDirect3DSurface = context.InputFrame.Direct3DSurface;
float anyDpi = 96.0f;
var width = context.OutputFrame.Direct3DSurface.Description.Width;
var height = context.OutputFrame.Direct3DSurface.Description.Height;
// Part 1: Apply effect using Imaging SDK
CanvasRenderTarget renderTarget = new CanvasRenderTarget(canvasDevice, (float)width, (float)height, anyDpi);
if (inputDirect3DSurface != null && renderTarget != null)
{
if (m_direct3DSurfaceImageSource == null)
{
m_direct3DSurfaceImageSource = new Direct3DSurfaceImageSource();
}
m_direct3DSurfaceImageSource.Direct3DSurface = inputDirect3DSurface;
((IImageConsumer)m_grayScaleEffect).Source = m_direct3DSurfaceImageSource;
if (m_direct3DSurfaceRenderer == null)
{
m_direct3DSurfaceRenderer = new Direct3DSurfaceRenderer();
}
m_direct3DSurfaceRenderer.Direct3DSurface = renderTarget;
m_direct3DSurfaceRenderer.Source = m_grayScaleEffect;
var task = m_direct3DSurfaceRenderer.RenderAsync().AsTask();
task.Wait();
m_direct3DSurfaceRenderer.Direct3DSurface = null;
m_direct3DSurfaceImageSource.Direct3DSurface = null;
}
// Part 2: Apply another effect using Win2D
var size = context.OutputFrame.Direct3DSurface.Description.Width * context.OutputFrame.Direct3DSurface.Description.Height;
var colors = Enumerable.Repeat(Colors.Black, size).ToArray();
var inputBitmap = CanvasBitmap.CreateFromColors(canvasDevice, colors, width, height, anyDpi);
inputBitmap.CopyPixelsFromBitmap(renderTarget);
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
float dispX = (float)Math.Cos(time.TotalSeconds) * 75f;
float dispY = (float)Math.Sin(time.TotalSeconds) * 75f;
ds.Clear(Colors.Black);
var dispMap = new DisplacementMapEffect()
{
Source = inputBitmap,
XChannelSelect = EffectChannelSelect.Red,
YChannelSelect = EffectChannelSelect.Green,
Amount = 100f,
Displacement = new Transform2DEffect()
{
TransformMatrix = Matrix3x2.CreateTranslation(dispX, dispY),
Source = new BorderEffect()
{
ExtendX = CanvasEdgeBehavior.Mirror,
ExtendY = CanvasEdgeBehavior.Mirror,
Source = new TurbulenceEffect()
{
Octaves = 3
}
}
}
};
ds.DrawImage(dispMap, -25f, -25f);
}
((IDisposable)renderTarget).Dispose();
((IDisposable)inputBitmap).Dispose();
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
LatestSoftwareBitmap = context.InputFrame.SoftwareBitmap;
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
if (context.InputFrame.SoftwareBitmap == null)
return;
var softwarebitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.OutputFrame.SoftwareBitmap.BitmapAlphaMode);
context.InputFrame.SoftwareBitmap.CopyTo(softwarebitmap);
recognizer.Treshold = (double)_configuration["tolerance"];
recognizer.Hue = (int)_configuration["hue"];
var result = recognizer.Recognize(softwarebitmap);
var s = string.Empty;
if (result != null)
{
for (int i = 0; i < result.GetLength(0); i++)
{
for (int j = 0; j < result.GetLength(1); j++)
{
var t = result[i, j];
s += t == 0 ? " 0" : t.ToString();
s += ' ';
}
s += Environment.NewLine;
}
}
if (result != null)
{
prevCount = 0;
var matrix = new int[10, 10]
{
{ 0, 0, -1, 0, 0, 0, 0, -1, 0, 0 },
{ 0, 0, -1, 0, 0, 0, 0, -1, 0, 0 },
{ -1, -1, -1, 0, 0, -1, 0, -1, 0, 0 },
{ 0, 0, -1, 0, -1, -1, -1, -1, -1, -1 },
{ 0, 0, -1, 0, 0, -1, 0, -1, 0, 0 },
{ 0, 0, -1, 0, 0, -1, 0, -1, 0, 0 },
{ 0, 0, -1, 0, 0, -1, 0, 0, 0, 0 },
{ -1, -1, -1, -1, -1, -1, 0, -1, 0, 0 },
{ 0, 0, -1, 0, 0, 0, 0, 0, 0, 0 },
{ 0, 0, -1, 0, 0, 0, 0, 0, 0, 0 },
};
var test = solver.Test(matrix, 10, 10);
char[,] solvedCrossword = null;
if (test)
{
solvedCrossword = solver.FirstTest(10, 10);
//var s = string.Empty;
//for (int i = 0; i < solvedCrossword.GetLength(0); i++)
//{
// for (int j = 0; j < solvedCrossword.GetLength(1); j++)
// {
// var t = solvedCrossword[i, j];
// s += (t == '\0') ? '*' : t;
// }
// s += Environment.NewLine;
//}
}
}
if (prevValue == recognizer.DetectedCenters.Count)
prevCount++;
_configuration["result"] = recognizer.DetectedCenters.Count;
_configuration["centers"] = recognizer.DetectedCenters;
prevValue = recognizer.DetectedCenters.Count;
softwarebitmap.CopyTo(context.OutputFrame.SoftwareBitmap);
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
//using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
//using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
//{
// var effect = new EdgeDetectionEffect
// {
// Source = inputBitmap,
// Amount = this.Amount
// };
// ds.DrawImage(effect);
//}
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var effect = new EdgeDetectionEffect
{
Source = inputBitmap,
Amount = this.Amount
};
ds.DrawImage(effect);
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
#if DEBUG
if (!debugOutputFlag)
{
Debug.WriteLine($"PixelFormat: {context.InputFrame.SoftwareBitmap.BitmapPixelFormat}");
debugOutputFlag = true;
}
#endif
// ********** Test for using SoftwareBitmap.Convert ********* //
//using (var inputBitmap = CanvasBitmap.CreateFromSoftwareBitmap(
// _canvasDevice,
// SoftwareBitmap.Convert(context.InputFrame.SoftwareBitmap, context.InputFrame.SoftwareBitmap.BitmapPixelFormat))) //PixelFormat: Bgra8
//using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (var ds = renderTarget.CreateDrawingSession())
//{
// var effect = new EdgeDetectionEffect
// {
// Source = inputBitmap,
// Amount = this.Amount
// };
// ds.DrawImage(effect);
//}
// ********************************************************** //
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var effect = new EdgeDetectionEffect
{
Source = inputBitmap,
Amount = this.Amount
};
ds.DrawImage(effect);
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
//using (var input = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
//using (var output = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (var ds = output.CreateDrawingSession())
//{
// var time = context.InputFrame.RelativeTime ?? new TimeSpan();
// ds.Clear(Colors.Black);
// for (uint i = 0; i < _numColumns; i++)
// {
// for (uint j = 0; j < _numRows; j++)
// {
// _crops[i, j].Source = input;
// float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
// float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
// Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile);
// _transforms[i, j].TransformMatrix =
// Matrix3x2.CreateRotation(rotation, centerPoint) *
// Matrix3x2.CreateScale(scale, centerPoint);
// ds.DrawImage(_transforms[i, j]);
// }
// }
//}
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var time = context.InputFrame.RelativeTime ?? new TimeSpan();
ds.Clear(Colors.Black);
for (uint i = 0; i < _numColumns; i++)
{
for (uint j = 0; j < _numRows; j++)
{
_crops[i, j].Source = inputBitmap;
float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile);
_transforms[i, j].TransformMatrix =
Matrix3x2.CreateRotation(rotation, centerPoint) *
Matrix3x2.CreateScale(scale, centerPoint);
ds.DrawImage(_transforms[i, j]);
}
}
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var time = context.InputFrame.RelativeTime ?? new TimeSpan();
ds.Clear(Colors.Black);
for (uint i = 0; i < _numColumns; i++)
{
for (uint j = 0; j < _numRows; j++)
{
_crops[i, j].Source = inputBitmap;
float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile);
_transforms[i, j].TransformMatrix =
Matrix3x2.CreateRotation(rotation, centerPoint) *
Matrix3x2.CreateScale(scale, centerPoint);
ds.DrawImage(_transforms[i, j]);
}
}
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputDirect3DSurface = context.InputFrame.Direct3DSurface;
float anyDpi = 96.0f;
var width = context.OutputFrame.Direct3DSurface.Description.Width;
var height = context.OutputFrame.Direct3DSurface.Description.Height;
// Part 1: Apply effect using Imaging SDK
CanvasRenderTarget renderTarget = new CanvasRenderTarget(canvasDevice, (float)width, (float)height, anyDpi);
if (inputDirect3DSurface != null && renderTarget != null)
{
if (m_direct3DSurfaceImageSource == null)
{
m_direct3DSurfaceImageSource = new Direct3DSurfaceImageSource();
}
m_direct3DSurfaceImageSource.Direct3DSurface = inputDirect3DSurface;
((IImageConsumer)m_grayScaleEffect).Source = m_direct3DSurfaceImageSource;
if (m_direct3DSurfaceRenderer == null)
{
m_direct3DSurfaceRenderer = new Direct3DSurfaceRenderer();
}
m_direct3DSurfaceRenderer.Direct3DSurface = renderTarget;
m_direct3DSurfaceRenderer.Source = m_grayScaleEffect;
var task = m_direct3DSurfaceRenderer.RenderAsync().AsTask();
task.Wait();
m_direct3DSurfaceRenderer.Direct3DSurface = null;
m_direct3DSurfaceImageSource.Direct3DSurface = null;
}
// Part 2: Apply another effect using Win2D
var size = context.OutputFrame.Direct3DSurface.Description.Width * context.OutputFrame.Direct3DSurface.Description.Height;
var colors = Enumerable.Repeat(Colors.Black, size).ToArray();
var inputBitmap = CanvasBitmap.CreateFromColors(canvasDevice, colors, width, height, anyDpi);
inputBitmap.CopyPixelsFromBitmap(renderTarget);
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
float dispX = (float)Math.Cos(time.TotalSeconds) * 75f;
float dispY = (float)Math.Sin(time.TotalSeconds) * 75f;
ds.Clear(Colors.Black);
var dispMap = new DisplacementMapEffect()
{
Source = inputBitmap,
XChannelSelect = EffectChannelSelect.Red,
YChannelSelect = EffectChannelSelect.Green,
Amount = 100f,
Displacement = new Transform2DEffect()
{
TransformMatrix = Matrix3x2.CreateTranslation(dispX, dispY),
Source = new BorderEffect()
{
ExtendX = CanvasEdgeBehavior.Mirror,
ExtendY = CanvasEdgeBehavior.Mirror,
Source = new TurbulenceEffect()
{
Octaves = 3
}
}
}
};
ds.DrawImage(dispMap, -25f, -25f);
}
((IDisposable)renderTarget).Dispose();
((IDisposable)inputBitmap).Dispose();
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
//using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
//using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
//{
// var grayscale = new GrayscaleEffect()
// {
// Source = inputBitmap
// };
// ds.DrawImage(grayscale);
//}
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var grayscale = new GrayscaleEffect()
{
Source = inputBitmap
};
ds.DrawImage(grayscale);
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var grayscale = new GrayscaleEffect()
{
Source = inputBitmap
};
ds.DrawImage(grayscale);
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
var inputFrameBitmap = context.InputFrame.SoftwareBitmap;
Snap = inputFrameBitmap;
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
processHandler(context, canvasDevice);
}
