public static CanvasBitmap Render(ICanvasResourceCreatorWithDpi resourceCreator, TextRenderSettings settings, Size renderSize, bool clip)
{
Rect geometryBounds;
ICanvasResourceCreator device;
CanvasRenderTarget geometryRender;
float dpi = 96;
if (resourceCreator == null)
{
device = new CanvasDevice();
geometryRender = new CanvasRenderTarget(device, (float)renderSize.Width, (float)renderSize.Height, dpi);
}
else
{
device = resourceCreator;
dpi = resourceCreator.Dpi;
geometryRender = new CanvasRenderTarget(resourceCreator, (float)renderSize.Width, (float)renderSize.Height);
}
// получаем сессию для рисования текста
using (var geometrySession = geometryRender.CreateDrawingSession())
{
geometrySession.Clear(Color.FromArgb(0, 0, 0, 0)); // прозрачность
var format = new CanvasTextFormat()
{
FontSize = settings.FontSize,
FontFamily = settings.FontFamily.Source,
HorizontalAlignment = CanvasHorizontalAlignment.Center,
VerticalAlignment = CanvasVerticalAlignment.Center,
};
// создаем тектовый слой
using (CanvasTextLayout textLayout = new CanvasTextLayout(device, settings.Text, format, (float)renderSize.Width, (float)renderSize.Height))
{
geometrySession.DrawTextLayout(textLayout, 0, 0, settings.TextColor);
CanvasGeometry geometry = CanvasGeometry.CreateText(textLayout);
CanvasStrokeStyle stroke = new CanvasStrokeStyle()
{
DashStyle = settings.OutlineTextSettings.DashStyle
};
geometryBounds = geometry.ComputeStrokeBounds(settings.OutlineTextSettings.Enabled ? settings.OutlineTextSettings.StrokeWidth : 0.0f);
if (settings.OutlineTextSettings.Enabled)
{
geometrySession.DrawGeometry(
geometry,
settings.OutlineTextSettings.StrokeColor,
settings.OutlineTextSettings.StrokeWidth,
stroke);
}
}
}
CanvasBitmap bitmap = CanvasBitmap.CreateFromDirect3D11Surface(device, geometryRender, dpi);
var computedWidth = (float)renderSize.Width;
var computedHeight = (float)renderSize.Height;
Matrix3x2 clipMatrix = Matrix3x2.Identity;
// Если требуется обрезка
if (clip)
{
var shadowOffset = settings.ShadowEffectSettings.BlurRadius * 2;
computedWidth = (float)Math.Floor(geometryBounds.Width + shadowOffset + Math.Abs(settings.ShadowEffectSettings.TranslationX));
computedHeight = (float)Math.Floor(geometryBounds.Height + shadowOffset + Math.Abs(settings.ShadowEffectSettings.TranslationY));
var offsetX = -(float)Math.Truncate(geometryBounds.Left);
var offsetY = -(float)Math.Truncate(geometryBounds.Top);
clipMatrix = Matrix3x2.CreateTranslation(new Vector2(offsetX, offsetY));
}
CanvasRenderTarget resultRender;
if (resourceCreator == null)
{
resultRender = new CanvasRenderTarget(device, computedWidth, computedHeight, dpi);
}
else
{
resultRender = new CanvasRenderTarget(resourceCreator, computedWidth, computedHeight);
}
// получаем сессию для рисования текста
using (var resultSession = resultRender.CreateDrawingSession())
{
resultSession.Clear(Color.FromArgb(0, 0, 0, 0));
var shadow = new ShadowEffect
{
Source = bitmap,
BlurAmount = settings.ShadowEffectSettings.BlurRadius
};
Transform2DEffect shadowTranslation = new Transform2DEffect
{
Source = shadow,
TransformMatrix = clipMatrix * Matrix3x2.CreateTranslation(new Vector2(
settings.ShadowEffectSettings.TranslationX,
settings.ShadowEffectSettings.TranslationY
))
};
resultSession.DrawImage(shadowTranslation);
Transform2DEffect imageTranslation = new Transform2DEffect
{
Source = bitmap,
TransformMatrix = clipMatrix
};
resultSession.DrawImage(imageTranslation);
}
return(CanvasBitmap.CreateFromDirect3D11Surface(device, resultRender));
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
//using (var input = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
//using (var output = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (var ds = output.CreateDrawingSession())
//{
// var time = context.InputFrame.RelativeTime ?? new TimeSpan();
// ds.Clear(Colors.Black);
// for (uint i = 0; i < _numColumns; i++)
// {
// for (uint j = 0; j < _numRows; j++)
// {
// _crops[i, j].Source = input;
// float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
// float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
// Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile);
// _transforms[i, j].TransformMatrix =
// Matrix3x2.CreateRotation(rotation, centerPoint) *
// Matrix3x2.CreateScale(scale, centerPoint);
// ds.DrawImage(_transforms[i, j]);
// }
// }
//}
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var time = context.InputFrame.RelativeTime ?? new TimeSpan();
ds.Clear(Colors.Black);
for (uint i = 0; i < _numColumns; i++)
{
for (uint j = 0; j < _numRows; j++)
{
_crops[i, j].Source = inputBitmap;
float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile);
_transforms[i, j].TransformMatrix =
Matrix3x2.CreateRotation(rotation, centerPoint) *
Matrix3x2.CreateScale(scale, centerPoint);
ds.DrawImage(_transforms[i, j]);
}
}
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var time = context.InputFrame.RelativeTime ?? new TimeSpan();
ds.Clear(Colors.Black);
for (uint i = 0; i < _numColumns; i++)
{
for (uint j = 0; j < _numRows; j++)
{
_crops[i, j].Source = inputBitmap;
float scale = Rescale((float)(Math.Cos(time.TotalSeconds * 2f + 0.2f * (i + j))), 0.6f, 0.95f);
float rotation = (float)time.TotalSeconds * 1.5f + 0.2f * (i + j);
Vector2 centerPoint = new Vector2((i + 0.5f) * PixelsPerTile, (j + 0.5f) * PixelsPerTile);
_transforms[i, j].TransformMatrix =
Matrix3x2.CreateRotation(rotation, centerPoint) *
Matrix3x2.CreateScale(scale, centerPoint);
ds.DrawImage(_transforms[i, j]);
}
}
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
//using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
//using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
//{
// var effect = new EdgeDetectionEffect
// {
// Source = inputBitmap,
// Amount = this.Amount
// };
// ds.DrawImage(effect);
//}
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var effect = new EdgeDetectionEffect
{
Source = inputBitmap,
Amount = this.Amount
};
ds.DrawImage(effect);
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
#if DEBUG
if (!debugOutputFlag)
{
Debug.WriteLine($"PixelFormat: {context.InputFrame.SoftwareBitmap.BitmapPixelFormat}");
debugOutputFlag = true;
}
#endif
// ********** Test for using SoftwareBitmap.Convert ********* //
//using (var inputBitmap = CanvasBitmap.CreateFromSoftwareBitmap(
// _canvasDevice,
// SoftwareBitmap.Convert(context.InputFrame.SoftwareBitmap, context.InputFrame.SoftwareBitmap.BitmapPixelFormat))) //PixelFormat: Bgra8
//using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (var ds = renderTarget.CreateDrawingSession())
//{
// var effect = new EdgeDetectionEffect
// {
// Source = inputBitmap,
// Amount = this.Amount
// };
// ds.DrawImage(effect);
//}
// ********************************************************** //
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var effect = new EdgeDetectionEffect
{
Source = inputBitmap,
Amount = this.Amount
};
ds.DrawImage(effect);
}
}
}
}
// ** Methods ** //
// This is run for every video frame passed in the media pipleine (MediaPlayer, MediaCapture, etc)
public void ProcessFrame(ProcessVideoFrameContext context)
{
evaluatableVideoFrame = VideoFrame.CreateWithDirect3D11Surface(context.InputFrame.Direct3DSurface);
// ********** Draw Bounding Boxes with Win2D ********** //
// Use Direct3DSurface if using GPU memory
if (context.InputFrame.Direct3DSurface != null)
{
if (modelBindingComplete && options.PreferredDeviceKind != LearningModelDeviceKindPreview.LearningDeviceGpu)
{
options.PreferredDeviceKind = LearningModelDeviceKindPreview.LearningDeviceGpu;
}
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
foreach (var box in filteredBoxes)
{
var x = (uint)Math.Max(box.X, 0);
var y = (uint)Math.Max(box.Y, 0);
var w = (uint)Math.Min(renderTarget.Bounds.Width - x, box.Width);
var h = (uint)Math.Min(renderTarget.Bounds.Height - y, box.Height);
// Draw the Text 10px above the top of the bounding box
ds.DrawText(box.Label, x, y - 10, Colors.Yellow);
ds.DrawRectangle(new Rect(x, y, w, h), new CanvasSolidColorBrush(canvasDevice, Colors.Yellow), 2f);
}
}
return;
}
// Use SoftwareBitmap if using CPU memory
if (context.InputFrame.SoftwareBitmap != null)
{
if (modelBindingComplete && options.PreferredDeviceKind != LearningModelDeviceKindPreview.LearningDeviceCpu)
{
options.PreferredDeviceKind = LearningModelDeviceKindPreview.LearningDeviceCpu;
}
// InputFrame's pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(canvasDevice, inputFrameBytes, context.InputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(canvasDevice, context.OutputFrame.SoftwareBitmap.PixelWidth, context.InputFrame.SoftwareBitmap.PixelHeight, (float)context.OutputFrame.SoftwareBitmap.DpiX, context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(), CanvasAlphaMode.Premultiplied))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
foreach (var box in filteredBoxes)
{
var x = (uint)Math.Max(box.X, 0);
var y = (uint)Math.Max(box.Y, 0);
var w = (uint)Math.Min(context.OutputFrame.SoftwareBitmap.PixelWidth - x, box.Width);
var h = (uint)Math.Min(context.OutputFrame.SoftwareBitmap.PixelHeight - y, box.Height);
// Draw the Text 10px above the top of the bounding box
ds.DrawText(box.Label, x, y - 10, Colors.Yellow);
ds.DrawRectangle(new Rect(x, y, w, h), new CanvasSolidColorBrush(canvasDevice, Colors.Yellow), 2f);
}
}
}
}
public async void SeparateThreadToSaveVideoStream()
{
while (isRecording == true)
{
//thread is counting
//sometimes stuck at thread at frame : 0, depending on the window chosen to record
//meaning OnFrameArrived is not called
if (parent != null)
{
DateTime currentTimeLocal = DateTime.Now;
TimeSpan elpasedTimeLocal = currentTimeLocal - initialRecordTime;
string debugstr = "At frame: " + counter.ToString() + " Threadcounter: " + threadcounter.ToString();
debugstr += " StreamSize: " + ((int)(videostream.Size / 1024.0)).ToString() + "KB TimeElapsed: " + ((int)elpasedTimeLocal.TotalSeconds).ToString();
parent.msg(debugstr);
}
threadcounter++;
if (threadcounter > 200000)
{
threadcounter = 0;
}
if (currentFrame != null)
{
///need to handle device lost
CanvasBitmap canvasBitmap = CanvasBitmap.CreateFromDirect3D11Surface(
canvasDevice,
currentFrame.Surface);
using (var inputstream = new InMemoryRandomAccessStream())
{
CancellationToken ct = new CancellationToken();
await canvasBitmap.SaveAsync(inputstream, CanvasBitmapFileFormat.Png, 1f).AsTask(ct);
ulong currentFrameLength = inputstream.Size;
_currentVideoStreamPos = videostream.Position;
await RandomAccessStream.CopyAsync(inputstream, videostream);
await videostream.FlushAsync(); //works, but significant slow down
DateTime currentTimeLocal = DateTime.Now;
TimeSpan diff = currentTimeLocal - previousRecordTime;
previousRecordTime = currentTimeLocal;
/// await RandomAccessStream.CopyAsync(inputstream, memorystream);
/// await memorystream.FlushAsync(); //works, but significant slow down
UnpackItem unpackItem = new UnpackItem();
unpackItem.pos = _currentVideoStreamPos;
unpackItem.length = currentFrameLength;
unpackItem.frameTime = diff;
unpackList.Add(unpackItem);
}
currentFrame?.Dispose();
currentFrame = null; //need this line so this thread will continue loop when new frame is not yet ready
}
else
{
Thread.Sleep(20);
}
}
await CloseVideoStream();
int len = unpackList.Count;
DateTime currentTime = DateTime.Now;
TimeSpan elpasedTime = currentTime - initialRecordTime;
string debugstrx = "Num frame: " + len.ToString() + " Threadcounter: " + threadcounter.ToString();
debugstrx += " StreamSize: " + ((int)(videostream.Size / 1024.0)).ToString() + "KB TimeElapsed: " + ((int)elpasedTime.TotalSeconds).ToString();
if (elpasedTime.TotalSeconds > 0)
{
debugstrx += " Frame Rate (fps) : " + (len / (double)elpasedTime.TotalSeconds).ToString();
}
if (parent != null)
{
parent.StartWritingReport(debugstrx);
}
//await parent.UnpackToMp4();
}
private Task CreateScreenCaptureTask()
{
Logger.Debug("Screen", "CreateScreenCaptureTask");
return(new Task(() => {
if (null == customVideoCapturer)
{
return;
}
while (true)
{
using (var frame = screenCaptureQueue.Take())
{
if (null == frame)
{
return;
}
if ((frame.ContentSize.Width != lastSize.Width) ||
(frame.ContentSize.Height != lastSize.Height))
{
lastSize = frame.ContentSize;
ResetFramePool(frame.ContentSize, false);
}
try
{
var canvasBitmap = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, frame.Surface);
uint actualBitmapWidth = canvasBitmap.SizeInPixels.Width;
uint actualBitmapHeight = canvasBitmap.SizeInPixels.Height;
uint bitmapWidth = actualBitmapWidth;
uint bitmapHeight = actualBitmapHeight;
if (bitmapWidth % 2 != 0)
{
bitmapWidth += 1;
}
if (bitmapHeight % 2 != 0)
{
bitmapHeight += 1;
}
VideoData rgbData = new VideoData((ulong)(bitmapWidth * bitmapHeight * 4));
var pixels = canvasBitmap.GetPixelBytes();
if (bitmapWidth != actualBitmapWidth)
{
var tmpPixels = new byte[bitmapWidth * bitmapHeight * 4];
Int64 indexSource = 0;
Int64 indexDest = 0;
Int64 strideSource = actualBitmapWidth * 4;
Int64 strideDest = bitmapWidth * 4;
for (uint y = 0; y < actualBitmapHeight; ++y)
{
Array.Copy(pixels, indexSource, tmpPixels, indexDest, strideSource);
indexSource += strideSource;
indexDest += strideDest;
}
pixels = tmpPixels;
}
rgbData.SetData8bit(pixels);
var buffer = VideoFrameBuffer.CreateFromARGB(
(int)bitmapWidth,
(int)bitmapHeight,
(int)(4 * bitmapWidth),
rgbData);
customVideoCapturer.NotifyFrame(
buffer,
(ulong)(DateTimeOffset.Now.ToUnixTimeMilliseconds()),
VideoRotation.Rotation0);
}
catch (Exception e) when(canvasDevice.IsDeviceLost(e.HResult))
{
ResetFramePool(frame.ContentSize, true);
}
}
}
}));
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
//using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
//using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
//using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
//{
// var grayscale = new GrayscaleEffect()
// {
// Source = inputBitmap
// };
// ds.DrawImage(grayscale);
//}
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var grayscale = new GrayscaleEffect()
{
Source = inputBitmap
};
ds.DrawImage(grayscale);
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var grayscale = new GrayscaleEffect()
{
Source = inputBitmap
};
ds.DrawImage(grayscale);
}
}
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
CanvasRenderTarget rt = new CanvasRenderTarget(canvasDevice, context.OutputFrame.Direct3DSurface.Description.Width, context.OutputFrame.Direct3DSurface.Description.Height, 96.0f);
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = rt.CreateDrawingSession())
{
TimeSpan time = context.InputFrame.RelativeTime.HasValue ? context.InputFrame.RelativeTime.Value : new TimeSpan();
float dispX = (float)Math.Cos(time.TotalSeconds) * 75f;
float dispY = (float)Math.Sin(time.TotalSeconds) * 75f;
ds.Clear(Colors.Black);
//var posterizeEffect = new PosterizeEffect()
//{
// Source = input,
// BlueValueCount = 2,
// RedValueCount = 2,
// GreenValueCount = 2,
//};
var transformEffect = new Transform2DEffect()
{
Source = input,
TransformMatrix = new System.Numerics.Matrix3x2(-1, 0, 0, 1, rt.SizeInPixels.Width, 0),
};
var discreteTransferEffect = new DiscreteTransferEffect()
{
Source = transformEffect,
RedTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f },
GreenTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f },
BlueTable = new float[] { 0.0f, 1.0f, 1.0f, 1.0f },
AlphaDisable = true,
};
var dispMap = new ConvolveMatrixEffect()
{
KernelMatrix = new float[]
{
0, +1, 0,
+1, -4, +1,
0, +1, 0,
},
Source = discreteTransferEffect,
Divisor = 0.2f,
};
var modEffect = new ArithmeticCompositeEffect()
{
Source1 = dispMap,
Source2 = dispMap,
MultiplyAmount = 1,
};
//var finalPosterizeEffect = new PosterizeEffect()
//{
// Source = modEffect,
// BlueValueCount = 2,
// RedValueCount = 2,
// GreenValueCount = 2,
//};
//var dispMap = new EdgeDetectionEffect()
//{
// Source = greyScaleEffect,
// Mode = EdgeDetectionEffectMode.Sobel,
//};
Rect src = new Rect(rt.SizeInPixels.Width / 3, rt.SizeInPixels.Height / 3, rt.SizeInPixels.Width / 3, rt.SizeInPixels.Height / 3);
//ds.DrawImage(input, bottomLeft);
//ds.DrawImage(discreteTransferEffect, bottomRight, src);
//ds.DrawImage(dispMap, topLeft, src);
ds.DrawImage(modEffect, src, src);
}
int centerX = 0;
int centerY = 0;
int cubletWidth = 0;
int cubletHeight = 0;
int analysisAreaX = (int)rt.SizeInPixels.Width * 3 / 10;
int analysisWidth = (int)rt.SizeInPixels.Width * 4 / 10;
byte[] analysisHorzBytes = rt.GetPixelBytes(analysisAreaX, (int)rt.SizeInPixels.Height / 2, analysisWidth, 1);
int analysisAreaY = (int)rt.SizeInPixels.Height * 3 / 10;
int analysisHeight = (int)rt.SizeInPixels.Height * 4 / 10;
byte[] analysisVertBytes = rt.GetPixelBytes((int)rt.SizeInPixels.Width / 2, analysisAreaY, 1, analysisHeight);
int foundLeft = 0;
for (int i = 0; i < analysisWidth / 2; i++)
{
byte b = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 0];
byte g = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 1];
byte r = analysisHorzBytes[4 * (analysisWidth / 2 - i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundLeft == 0)
{
foundLeft = i;
}
}
int foundRight = 0;
for (int i = 0; i < analysisWidth / 2; i++)
{
byte r = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 0];
byte g = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 1];
byte b = analysisHorzBytes[4 * (analysisWidth / 2 + i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundRight == 0)
{
foundRight = i;
}
}
int foundTop = 0;
for (int i = 0; i < analysisHeight / 2; i++)
{
byte r = analysisVertBytes[4 * (analysisHeight / 2 - i) + 0];
byte g = analysisVertBytes[4 * (analysisHeight / 2 - i) + 1];
byte b = analysisVertBytes[4 * (analysisHeight / 2 - i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundTop == 0)
{
foundTop = i;
}
}
int foundBottom = 0;
for (int i = 0; i < analysisHeight / 2; i++)
{
byte r = analysisVertBytes[4 * (analysisHeight / 2 + i) + 0];
byte g = analysisVertBytes[4 * (analysisHeight / 2 + i) + 1];
byte b = analysisVertBytes[4 * (analysisHeight / 2 + i) + 2];
if ((r > 100 || g > 100 || b > 50) && foundBottom == 0)
{
foundBottom = i;
}
}
centerX = (-foundLeft + foundRight) / 2 + (int)rt.SizeInPixels.Width / 2;
centerY = (-foundTop + foundBottom) / 2 + (int)rt.SizeInPixels.Height / 2;
cubletWidth = (int)((foundLeft + foundRight) * 1.2f);
cubletHeight = (int)((foundTop + foundBottom) * 1.2f);
// No 2d arrays in WinRT components? Boo.
// "Error C1113 #using failed on 'rubikscuberecognitionwinrt.winmd' RubiksCubeRecognitionLib
Vector2[] cubletCenters = new Vector2[3 * 3];
cubletCenters[1 * 3 + 1] = new Vector2(centerX, centerY);
cubletCenters[1 * 3 + 0] = new Vector2(centerX, centerY - cubletHeight);
cubletCenters[1 * 3 + 2] = new Vector2(centerX, centerY + cubletHeight);
cubletCenters[0 * 3 + 1] = new Vector2(centerX - cubletWidth, centerY);
cubletCenters[0 * 3 + 0] = new Vector2(centerX - cubletWidth, centerY - cubletHeight);
cubletCenters[0 * 3 + 2] = new Vector2(centerX - cubletWidth, centerY + cubletHeight);
cubletCenters[2 * 3 + 1] = new Vector2(centerX + cubletWidth, centerY);
cubletCenters[2 * 3 + 0] = new Vector2(centerX + cubletWidth, centerY - cubletHeight);
cubletCenters[2 * 3 + 2] = new Vector2(centerX + cubletWidth, centerY + cubletHeight);
cubeletColorsMutex.WaitOne();
using (CanvasBitmap input = CanvasBitmap.CreateFromDirect3D11Surface(canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget output = CanvasRenderTarget.CreateFromDirect3D11Surface(canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = output.CreateDrawingSession())
{
var transformEffect = new Transform2DEffect()
{
Source = input,
TransformMatrix = new System.Numerics.Matrix3x2(-1, 0, 0, 1, output.SizeInPixels.Width, 0),
};
Rect src = new Rect(0, 0, output.SizeInPixels.Width, output.SizeInPixels.Height);
ds.DrawImage(transformEffect, src, src);
// Draw a crosshair on the screen
ds.FillRectangle(new Rect(output.SizeInPixels.Width / 2 - 3, output.SizeInPixels.Height / 4, 6, output.SizeInPixels.Height / 2), Colors.Gray);
ds.FillRectangle(new Rect(output.SizeInPixels.Width / 4, output.SizeInPixels.Height / 2 - 3, output.SizeInPixels.Width / 2, 6), Colors.Gray);
if (true)
{
for (int x = 0; x < 3; x++)
{
for (int y = 0; y < 3; y++)
{
int sampleWidth = 2;
byte[] cubletBytes = input.GetPixelBytes((int)cubletCenters[(2 - x) * 3 + y].X - sampleWidth / 2, (int)cubletCenters[(2 - x) * 3 + y].Y - sampleWidth / 2, sampleWidth, sampleWidth);
int totalR = 0; int totalG = 0; int totalB = 0;
for (int i = 0; i < sampleWidth * sampleWidth; i++)
{
totalB += cubletBytes[4 * i + 0];
totalG += cubletBytes[4 * i + 1];
totalR += cubletBytes[4 * i + 2];
}
cubletColors[x * 3 + y] = Color.FromArgb(255, (byte)(totalR / (sampleWidth * sampleWidth)), (byte)(totalG / (sampleWidth * sampleWidth)), (byte)(totalB / (sampleWidth * sampleWidth)));
}
}
for (int x = 0; x < 3; x++)
{
for (int y = 0; y < 3; y++)
{
ds.FillRectangle((float)cubletCenters[x * 3 + y].X - 12, (float)cubletCenters[x * 3 + y].Y - 12, 24, 24, Colors.Black);
ds.FillRectangle((float)cubletCenters[x * 3 + y].X - 10, (float)cubletCenters[x * 3 + y].Y - 10, 20, 20, cubletColors[x * 3 + y]);
}
}
}
}
cubeletColorsMutex.ReleaseMutex();
}
public async void ProcessFrame(ProcessVideoFrameContext context)
{
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
var chromaKeyEffect = new ChromaKeyEffect
{
Color = ChromaColor,
Source = inputBitmap,
Tolerance = Tolerance,
Feather = true
};
//only load bg image if it hasnt already been loaded or if image is different
if (_backgroundBitmap == null || _lastUsedImageFilePath != _imageFileName)
{
_backgroundBitmap = await CanvasBitmap.LoadAsync(ds, new Uri($"ms-appdata:///Local/{ImageFileName}"));
}
_lastUsedImageFilePath = _imageFileName; //keep track of any incoming image changes
var compositeEffect = new CompositeEffect
{
Sources = { _backgroundBitmap, chromaKeyEffect }
};
ds.DrawImage(compositeEffect);
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Ignore))
{
using (var ds = renderTarget.CreateDrawingSession())
{
var chroma = new ChromaKeyEffect
{
Color = ChromaColor,
Source = inputBitmap,
Tolerance = Tolerance
};
//only load bg image if it hasnt already been loaded or if image is different
if (_backgroundBitmap == null || _lastUsedImageFilePath != _imageFileName)
{
_backgroundBitmap = await CanvasBitmap.LoadAsync(ds, new Uri($"ms-appdata:///Local/{ImageFileName}"));
}
//keep track of any incoming image changes
_lastUsedImageFilePath = _imageFileName;
var compositeEffect = new CompositeEffect {
Mode = CanvasComposite.Add
};
compositeEffect.Sources.Add(_backgroundBitmap);
compositeEffect.Sources.Add(chroma);
ds.DrawImage(compositeEffect);
}
}
}
}
//private bool correctionFlag = false;
public void ProcessFrame(ProcessVideoFrameContext context)
{
using (CanvasBitmap inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (CanvasRenderTarget renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (CanvasDrawingSession ds = renderTarget.CreateDrawingSession())
using (var scaleEffect = new ScaleEffect())
using (CanvasSolidColorBrush solidColorBrush = new CanvasSolidColorBrush(_canvasDevice, _backgroundColor))
{
solidColorBrush.Opacity = _backgroundOpacity;
double rel = context.InputFrame.RelativeTime.Value.Ticks / (double)TimeSpan.TicksPerMillisecond;
//context.OutputFrame.Duration = new TimeSpan( (long)(frameLength * TimeSpan.TicksPerMillisecond));
int frameTimeCounter = (int)Math.Round(rel / _frameLength, 0);
int[] pitch = new int[_count];
int[] yaw = new int[_count];
int[] fov = new int[_count];
for (int i = 0; i < _count; i++)
{
try
{
//pitch[i] = this.pitch[ (frameTimeCounter + (int)Math.Round(offset, 0)) * (count) + i];
//fov[i] = this.fov[ (frameTimeCounter + (int)Math.Round(offset, 0)) * (count) + i];
//yaw[i] = this.yaw[ (frameTimeCounter + (int)Math.Round(offset, 0)) * (count) + i];
pitch[i] = this._pitch[(frameTimeCounter + (int)_offset) * (_count) + i];
fov[i] = this._fov[(frameTimeCounter + (int)_offset) * (_count) + i];
yaw[i] = this._yaw[(frameTimeCounter + (int)_offset) * (_count) + i];
}
catch (ArgumentOutOfRangeException ex)
{
Debug.WriteLine(ex.Message);
pitch[i] = 0;
fov[i] = 0;
yaw[i] = 0;
}
}
byte[] tab = Heatmap.GenerateHeatmap(pitch, yaw, fov);
CanvasBitmap cb = CanvasBitmap.CreateFromBytes(_canvasDevice, tab, 64, 64, Windows.Graphics.DirectX.DirectXPixelFormat.B8G8R8A8UIntNormalized, 96, CanvasAlphaMode.Premultiplied);
scaleEffect.Source = cb;
scaleEffect.Scale = new System.Numerics.Vector2((float)_width / 64, (float)_height / 64);
scaleEffect.InterpolationMode = CanvasImageInterpolation.Cubic;
scaleEffect.BorderMode = EffectBorderMode.Hard;
if (_graysclaleVideoFlag)
{
var grayScaleEffect = new GrayscaleEffect
{
BufferPrecision = CanvasBufferPrecision.Precision8UIntNormalized,
CacheOutput = false,
Source = inputBitmap
};
ds.DrawImage(grayScaleEffect);
}
else
{
ds.DrawImage(inputBitmap);
}
ds.DrawImage(scaleEffect, 0, 0, new Windows.Foundation.Rect {
Height = _height, Width = _width
}, _heatmapOpacity);
if (_generateDots)
{
for (int i = 0; i < _count; i++)
{
ds.FillCircle(yaw[i] * _width / 64, pitch[i] * _height / 64, _dotsRadius, _colors[i % 5]);
}
}
ds.FillRectangle(new Windows.Foundation.Rect {
Height = _height, Width = _width
}, solidColorBrush);
ds.Flush();
}
}
public void ProcessFrame(ProcessVideoFrameContext context)
{
try
{
// When using SupportedMemoryTypes => MediaMemoryTypes.GpuAndCpu we need to check if we're using GPU or CPU for the frame
// If we're on GPU, use InputFrame.Direct3DSurface
if (context.InputFrame.SoftwareBitmap == null)
{
using (var inputBitmap = CanvasBitmap.CreateFromDirect3D11Surface(_canvasDevice, context.InputFrame.Direct3DSurface))
using (var renderTarget = CanvasRenderTarget.CreateFromDirect3D11Surface(_canvasDevice, context.OutputFrame.Direct3DSurface))
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
ds.DrawImage(this._overlay, inputBitmap.Bounds);
}
return;
}
// If we're on CPU, use InputFrame.SoftwareBitmap
if (context.InputFrame.Direct3DSurface == null)
{
// InputFrame's raw pixels
byte[] inputFrameBytes = new byte[4 * context.InputFrame.SoftwareBitmap.PixelWidth * context.InputFrame.SoftwareBitmap.PixelHeight];
context.InputFrame.SoftwareBitmap.CopyToBuffer(inputFrameBytes.AsBuffer());
using (var inputBitmap = CanvasBitmap.CreateFromBytes(
_canvasDevice,
inputFrameBytes,
context.InputFrame.SoftwareBitmap.PixelWidth,
context.InputFrame.SoftwareBitmap.PixelHeight,
context.InputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat()))
using (var renderTarget = new CanvasRenderTarget(
_canvasDevice,
context.OutputFrame.SoftwareBitmap.PixelWidth,
context.OutputFrame.SoftwareBitmap.PixelHeight,
(float)context.OutputFrame.SoftwareBitmap.DpiX,
context.OutputFrame.SoftwareBitmap.BitmapPixelFormat.ToDirectXPixelFormat(),
CanvasAlphaMode.Premultiplied))
{
using (var ds = renderTarget.CreateDrawingSession())
{
ds.DrawImage(inputBitmap);
ds.DrawImage(this._overlay, inputBitmap.Bounds);
}
}
}
}
catch (Exception)
{
if (crashCount < 20)
{
crashCount++;
Debug.WriteLine($"ProcessFrame Exception: #{crashCount}");
}
else
{
//System.Exception HResult = 0x88990012
//Message = Objects used together must be created from the same factory instance. (Exception from HRESULT: 0x88990012)
//Source = System.Private.CoreLib
//StackTrace:
//at System.Runtime.InteropServices.WindowsRuntime.IClosable.Close()
//at System.Runtime.InteropServices.WindowsRuntime.IClosableToIDisposableAdapter.Dispose()
//at VideoEffects.Win2D.OverlayVideoEffect.ProcessFrame(ProcessVideoFrameContext context) in D:\GitHub\VideoDiary\src\VideoDiary.EffectsLibrary\Win2dEffects\OverlayVideoEffect.cs:line 66
throw;
}
}
}
private void ProcessFrame(Direct3D11CaptureFrame frame)
{
// Resize and device-lost leverage the same function on the
// Direct3D11CaptureFramePool. Refactoring it this way avoids
// throwing in the catch block below (device creation could always
// fail) along with ensuring that resize completes successfully and
// isn’t vulnerable to device-lost.
bool needsReset = false;
bool recreateDevice = false;
if ((frame.ContentSize.Width != _lastSize.Width) ||
(frame.ContentSize.Height != _lastSize.Height))
{
needsReset = true;
_lastSize = frame.ContentSize;
_swapChain.ResizeBuffers(_lastSize.Width, _lastSize.Height);
}
Direct3D11CaptureFrame direct = frame;
try
{
// Take the D3D11 surface and draw it into a
// Composition surface.
if (direct.SystemRelativeTime - lastFrameTime < TimeSpan.FromSeconds(1))
{
//F**k Microsoft🤬
MediaClip mediaClip = MediaClip.CreateFromSurface(direct.Surface, direct.SystemRelativeTime - lastFrameTime);
composition.Clips.Add(mediaClip);
}
lastFrameTime = direct.SystemRelativeTime;
// Convert our D3D11 surface into a Win2D object.
canvasBitmap = CanvasBitmap.CreateFromDirect3D11Surface(
_canvasDevice,
direct.Surface);
using (var drawingSession = _swapChain.CreateDrawingSession(Colors.Transparent))
{
//drawingSession.DrawCircle(400, 300, 100, Colors.Red, 20);
ScaleEffect effect = new ScaleEffect()
{
Source = canvasBitmap,
Scale = new Vector2((float)swapChain.ActualWidth / _item.Size.Width)
};
drawingSession.DrawImage(effect);
}
_swapChain.Present();
//canvasControl.Invalidate();
// Helper that handles the drawing for us, not shown.
}
// This is the device-lost convention for Win2D.
catch (Exception e) when(_canvasDevice.IsDeviceLost(e.HResult))
{
// We lost our graphics device. Recreate it and reset
// our Direct3D11CaptureFramePool.
needsReset = true;
recreateDevice = true;
}
if (needsReset)
{
ResetFramePool(direct.ContentSize, recreateDevice);
}
}
