public static async Task <SoftwareBitmap> ResizeSoftwareBitmap(SoftwareBitmap softwareBitmap, double scaleFactor)
{
var resourceCreator = CanvasDevice.GetSharedDevice();
var canvasBitmap = CanvasBitmap.CreateFromSoftwareBitmap(resourceCreator, softwareBitmap);
var canvasRenderTarget = new CanvasRenderTarget(resourceCreator, (int)(softwareBitmap.PixelWidth * scaleFactor), (int)(softwareBitmap.PixelHeight * scaleFactor), 96);
using (var cds = canvasRenderTarget.CreateDrawingSession())
{
cds.DrawImage(canvasBitmap, canvasRenderTarget.Bounds);
}
var pixelBytes = canvasRenderTarget.GetPixelBytes();
var writeableBitmap = new WriteableBitmap((int)(softwareBitmap.PixelWidth * scaleFactor), (int)(softwareBitmap.PixelHeight * scaleFactor));
using (var stream = writeableBitmap.PixelBuffer.AsStream())
{
await stream.WriteAsync(pixelBytes, 0, pixelBytes.Length);
}
var scaledSoftwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Rgba16, (int)(softwareBitmap.PixelWidth * scaleFactor), (int)(softwareBitmap.PixelHeight * scaleFactor));
scaledSoftwareBitmap.CopyFromBuffer(writeableBitmap.PixelBuffer);
return(scaledSoftwareBitmap);
}
public static async Task <SoftwareBitmap> RotateLeftAsync(this SoftwareBitmap source)
{
if (source == null)
{
return(null);
}
using (var memory = new InMemoryRandomAccessStream())
{
// BitmapEncoder を用いメモリ上で source をリサイズ
var id = BitmapEncoder.PngEncoderId;
var encoder = await BitmapEncoder.CreateAsync(id, memory);
encoder.BitmapTransform.Rotation = BitmapRotation.Clockwise270Degrees;
// encoder.BitmapTransform.InterpolationMode = BitmapInterpolationMode.Fant;
encoder.SetSoftwareBitmap(source);
await encoder.FlushAsync();
// リサイズしたメモリを WriteableBitmap に複写
var writeableBitmap = new WriteableBitmap(source.PixelHeight, source.PixelWidth);
await writeableBitmap.SetSourceAsync(memory);
// dest（XAML の Image コントロール互換）を作成し、WriteableBitmap から複写
var dest = new SoftwareBitmap(BitmapPixelFormat.Bgra8, writeableBitmap.PixelWidth, writeableBitmap.PixelHeight, BitmapAlphaMode.Premultiplied);
dest.CopyFromBuffer(writeableBitmap.PixelBuffer);
return(dest);
}
}
public async Task <StorageFile> WriteableBitmapSaveToFile(WriteableBitmap combine_wb)
{
if (combine_wb == null)
{
return(null);
}
var storageManager = await LocalCacheManager.InitializeAsync(StorageFolderType.Pictures);
var filename = "ink" + SecurityHelper.MD5(DateTime.Now.ToString(("yyyy-MM-dd HH:mm:ss fff"))) + ".jpg";
var path = Path.Combine(storageManager.CurrentFolder.Path, filename);
var md5Name = DownloadHelper.GetDownloadedLocalFileName(path);
StorageFile file = await storageManager.CurrentFolder.CreateFileAsync(md5Name, CreationCollisionOption.ReplaceExisting);
using (var fileStream = await file.OpenAsync(FileAccessMode.ReadWrite))
{
//await renderTarget.SaveAsync(fileStream, CanvasBitmapFileFormat.Png, 1f);
SoftwareBitmap softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, combine_wb.PixelWidth, combine_wb.PixelHeight);
softwareBitmap.CopyFromBuffer(combine_wb.PixelBuffer);
BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, fileStream);
encoder.SetSoftwareBitmap(softwareBitmap);
await encoder.FlushAsync();
}
return(file);
}
public async void ToWriteableBitmap()
{
SoftwareBitmap softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Gray8, (int)width, (int)height);
softwareBitmap.CopyFromBuffer(pgmByteData.AsBuffer());
if (softwareBitmap.BitmapPixelFormat != BitmapPixelFormat.Bgra8 ||
softwareBitmap.BitmapAlphaMode == BitmapAlphaMode.Straight)
{
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
}
bitmapImageSource = new SoftwareBitmapSource();
await bitmapImageSource.SetBitmapAsync(softwareBitmap);
//Save softwareBitmap to file
//using (IRandomAccessStream stream = await outputFile.OpenAsync(FileAccessMode.ReadWrite))
//{
// // Create an encoder with the desired format
// BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, stream);
// // Set the software bitmap
// encoder.SetSoftwareBitmap(softwareBitmap);
// await encoder.FlushAsync();
//}
}
private SoftwareBitmap SmoothImage(SoftwareBitmap softwareBitmap)
{
var pixels = new ImageData(softwareBitmap);
for (int height = 0; height < softwareBitmap.PixelHeight; height += SmoothHeight)
{
for (int width = 0; width < softwareBitmap.PixelWidth; width += SmoothWidth)
{
var r = new Task(() => pixels.red = getAvg(ref pixels.red, height, width, softwareBitmap));
var g = new Task(() => pixels.green = getAvg(ref pixels.green, height, width, softwareBitmap));
var b = new Task(() => pixels.blue = getAvg(ref pixels.blue, height, width, softwareBitmap));
r.Start();
g.Start();
b.Start();
r.Wait();
g.Wait();
b.Wait();
}
}
softwareBitmap.CopyFromBuffer(pixels.ToBuffer().AsBuffer());
return(softwareBitmap);
}
public async Task RenderAsync(Image <Rgba32> image, CancellationToken token)
{
Debug.WriteLine("Begin render");
if (bitmap == null || bitmap.PixelHeight != image.Height || bitmap.PixelWidth != image.Width)
{
//Image = new SoftwareBitmapSource();
bitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, image.Width, image.Height, BitmapAlphaMode.Premultiplied);
}
using var buffer = MemoryOwner <Bgra32> .Allocate(bitmap.PixelWidth *bitmap.PixelHeight);
Configuration configuration = new Configuration()
{
PreferContiguousImageBuffers = true
};
image.CloneAs <Bgra32>(configuration).DangerousTryGetSinglePixelMemory(out var bgraBuffer);
var tmpBuffer = MemoryMarshal.AsBytes(bgraBuffer.Span).ToArray();
bitmap.CopyFromBuffer(tmpBuffer.AsBuffer());
await Task.Factory.StartNew(() =>
{
Image.SetBitmapAsync(bitmap);
Debug.WriteLine("End render");
}, Task.Factory.CancellationToken, TaskCreationOptions.None, App.Current.UIScheduler);
}
public async Task <SoftwareBitmap> SaveToBitmap(InkStrokeContainer ink)
{
ICanvasResourceCreator device = CanvasDevice.GetSharedDevice();
CanvasRenderTarget renderTarget = new CanvasRenderTarget(device, Width, Height, 96);
var src = new byte[Width * 4 * Height];
for (int index = 0; index < src.Length; ++index)
{
src[index] = 255;
}
renderTarget.SetPixelBytes(src);
byte[] imageBytes = new byte[4 * Width * Height];
using (var ds = renderTarget.CreateDrawingSession())
{
IReadOnlyList <InkStroke> inklist = ink.GetStrokes();
ds.DrawInk(inklist);
}
var inkpixel = renderTarget.GetPixelBytes();
WriteableBitmap bmp = new WriteableBitmap(Width, Height);
Stream s = bmp.PixelBuffer.AsStream();
s.Seek(0, SeekOrigin.Begin);
s.Write(inkpixel, 0, Width * 4 * Height);
s.Position = 0;
await s.ReadAsync(imageBytes, 0, (int)s.Length);
SoftwareBitmap result = new SoftwareBitmap(BitmapPixelFormat.Bgra8, Width, Height, BitmapAlphaMode.Premultiplied);
result.CopyFromBuffer(imageBytes.AsBuffer());
return(result);
}
async private void Timer_Tick(object sender, object e)
{
if (timer_tick_complete_flag == 1)
{
return;
}
timer_tick_complete_flag = 1;
/* stream server */
if (streamSocketSrv.receive_buf_flag == 1)
{
receviebitMap.CopyFromBuffer(streamSocketSrv.receiverbuf);
await sbSource.SetBitmapAsync(receviebitMap);
streamSocketSrv.receive_buf_flag = 0;
client_send_serverip_flag = 0;
}
if (streamSocketClient.flag_client_start == 0)
{
await streamSocketClient.start(clientip, "22333");
client_send_serverip_flag = 1;
}
else
{
if (client_send_serverip_flag == 1)
{
await streamSocketClient.sendmsgString(serverip);
}
}
timer_tick_complete_flag = 0;
}
/// <summary>
/// converts the pixel data to a bitmap object
/// </summary>
/// <returns></returns>
public SoftwareBitmap ToSoftwareBitmap()
{
var bmp = new SoftwareBitmap(BitmapPixelFormat.Rgba8, Width, Height, BitmapAlphaMode.Ignore);
bmp.CopyFromBuffer(this.ToBitmap().PixelBuffer);
return(bmp);
}
private async void mergePhotos()
{
if (complete == true)
{
try
{
WriteableBitmap outputImage = new WriteableBitmap(images[0].PixelWidth * 2, images[0].PixelHeight * (totpics + 1));
WriteableBitmap inputImage = new WriteableBitmap(images[0].PixelWidth, images[0].PixelHeight);
const int BYTES_PER_PIXEL = 4;
SoftwareBitmap outputImage2 = new SoftwareBitmap(images[0].BitmapPixelFormat, images[0].PixelWidth * 2, images[0].PixelHeight * (totpics + 1));
int owidth = outputImage.PixelWidth;
int oheight = outputImage.PixelHeight;
int iwidth = images[0].PixelWidth;
int iheight = images[0].PixelHeight;
IBuffer inputbuffer = inputImage.PixelBuffer;
int bytesperpixel = 4;
int istride = iwidth * bytesperpixel;
int ostride = owidth * bytesperpixel;
byte[] imgdata = new byte[owidth * oheight * bytesperpixel];
byte[] inputdata;
int row = 0;
for (int i = 0; i < totpics + 1; i++)
{
images[i].CopyToBuffer(inputbuffer);
inputdata = inputbuffer.ToArray();
while (row < (iheight * (i + 1)))
{
for (int col = 0; col < iwidth; col++)
{
// BGRA
//set left image
imgdata[row * ostride + col * 4 + 0] = inputdata[(row - (iheight * i)) * istride + col * 4 + 0];
imgdata[row * ostride + col * 4 + 1] = inputdata[(row - (iheight * i)) * istride + col * 4 + 1];
imgdata[row * ostride + col * 4 + 2] = inputdata[(row - (iheight * i)) * istride + col * 4 + 2];
imgdata[row * ostride + col * 4 + 3] = inputdata[(row - (iheight * i)) * istride + col * 4 + 3];
//set right image
imgdata[row * ostride + (col + iwidth) * 4 + 0] = inputdata[(row - (iheight * i)) * istride + col * 4 + 0];
imgdata[row * ostride + (col + iwidth) * 4 + 1] = inputdata[(row - (iheight * i)) * istride + col * 4 + 1];
imgdata[row * ostride + (col + iwidth) * 4 + 2] = inputdata[(row - (iheight * i)) * istride + col * 4 + 2];
imgdata[row * ostride + (col + iwidth) * 4 + 3] = inputdata[(row - (iheight * i)) * istride + col * 4 + 3];
}
row++;
}
}
outputImage2.CopyFromBuffer(imgdata.AsBuffer());
StorageFile imageFile = await savePicturesFolder.CreateFileAsync("pb" + DateTime.Now.ToString("ssmmHHddMMyy") + ".bmp", CreationCollisionOption.ReplaceExisting);
System.Diagnostics.Debug.WriteLine("pb" + DateTime.Now.ToString("ssmmHHddMMyy") + ".bmp");
SaveSoftwareBitmapToFile(outputImage2, imageFile);
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine("Error Merging Photos");
System.Diagnostics.Debug.WriteLine(ex.ToString());
}
}
}
public SoftwareBitmap GetSoftwareBitmap()
{
var pixels = this.Pixels;
var w = this.PixelWidth;
var h = this.PixelHeight;
var bmp = new SoftwareBitmap(BitmapPixelFormat.Bgra8, w, h, BitmapAlphaMode.Premultiplied);
bmp.CopyFromBuffer(pixels.AsBuffer());
return(bmp);
}
public static async Task <SoftwareBitmap> CaptureVisualAsync(this Visual visual, uint captureWidth = 0, uint captureHeight = 0)
{
var captureTest = visual.GetCompositionCaptureTest();
uint width = captureWidth;
uint height = captureHeight;
if (width == 0 || height == 0)
{
throw new NotImplementedException("Query height and width from Visual");
}
// Render Visual:
// * This function is basically async and returns immediately after putting
// a MILCMD onto the batch for the application channel, and marks the device dirty.
// * It returns two handles by reference.
// * The first handle (hMap) is to a map of bits.
// * The second handle (hEvent) is to an event.
// * The event is signaled when a commit has happened
// and the after actual renderpass has been rendered and presented.
// * Once signaled, the bits are ready for us to grab.
// * Any changes to the tree before the implicit commit sends the batch to the
// Compositor will be reflected in the capture, even if made after the initial
// RenderVisual function call.
// * Any changes to the tree after the implicit commit may safely modify the tree,
// as they will be processed in a separate batch
IntPtr hMap = IntPtr.Zero;
IntPtr hEvent = IntPtr.Zero;
uint cbMap;
int hr = captureTest.RenderVisual(
visual,
0, // offset X
0, // offset y
width,
height,
(uint)BitmapPixelFormat.Bgra8,
ref hMap,
ref hEvent,
out cbMap);
if (hr != 0)
{
throw new Exception("Render Visual Failed, ErrCode: " + hr);
}
// Block waiting for the bits in a worker thread and return them as a SoftwareBitmap
var bitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, (int)width, (int)height);
var buffer = await Task.Run(() => GetVisualPixelBuffer(hEvent, hMap, cbMap));
bitmap.CopyFromBuffer(buffer);
return(bitmap);
}
private async void RenderImage(NotificationMessageAction <SoftwareBitmap> notificationMessageAction)
{
if (BaumCollection != null && BaumCollection.Count > 0)
{
if (BaumCollection.Where(x => x.IsMarked == true).Count() > 0)
{
SelectTree(BaumCollection.Where(x => x.IsMarked == true).First());
}
//_selectedBaum = _BaumCollection.Where(x => x.IsMarked = true).First();
//RaisePropertyChanged(() => SelectedBaum);
}
RenderTargetBitmap renderTargetBitmap = new RenderTargetBitmap();
await renderTargetBitmap.RenderAsync(mapGrid);
if (renderTargetBitmap.PixelHeight != 0 && renderTargetBitmap.PixelWidth != 0)
{
SoftwareBitmap softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, renderTargetBitmap.PixelWidth, renderTargetBitmap.PixelHeight);
softwareBitmap.DpiX = 600;
softwareBitmap.DpiY = 600;
softwareBitmap.CopyFromBuffer(await renderTargetBitmap.GetPixelsAsync());
notificationMessageAction.Execute(softwareBitmap);
}
//BitmapDecoder imagedecoder;
//using (var imagestream = await Map.OpenAsync(FileAccessMode.Read))
//{
// imagedecoder = await BitmapDecoder.CreateAsync(imagestream);
// CanvasDevice device = CanvasDevice.GetSharedDevice();
// CanvasRenderTarget renderTarget = new CanvasRenderTarget(device, imagedecoder.PixelWidth, imagedecoder.PixelHeight, 96);
// using (var ds = renderTarget.CreateDrawingSession())
// {
// ds.Clear(Colors.White);
// CanvasBitmap image = await CanvasBitmap.LoadAsync(device, imagestream);
// ds.DrawImage(image);
// //ds.DrawText(lblName.Text, new System.Numerics.Vector2(150, 150), Colors.Black);
// }
// await renderTarget.SaveAsync(imagestream, CanvasBitmapFileFormat.Jpeg);
// BitmapImage bitmap = new BitmapImage();
// bitmap.SetSource(imagestream);
// MapR = bitmap;
//}
}
public static SoftwareBitmap WriteableBitmapToSoftwareBitmap(WriteableBitmap writeablebitmap)
{
if (writeablebitmap == null)
{
return(null);
}
SoftwareBitmap softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, writeablebitmap.PixelWidth, writeablebitmap.PixelHeight);
softwareBitmap.CopyFromBuffer(writeablebitmap.PixelBuffer);
return(softwareBitmap);
}
void createWorker()
{
//create worker thread for reading barcode
readerWorker = new Task(async() =>
{
//use stopwatch to time the execution, and execute the reading process repeatedly
var watch = System.Diagnostics.Stopwatch.StartNew();
var reader = new QRCodeMultiReader();
SoftwareBitmap bitmap;
HybridBinarizer binarizer;
while (true)
{
try
{
lock (bufLock)
{
bitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, width, height);
bitmap.CopyFromBuffer(decodedDataBuf.AsBuffer());
}
}
catch
{
//the size maybe incorrect due to unknown reason
await Task.Delay(10);
continue;
}
var source = new SoftwareBitmapLuminanceSource(bitmap);
binarizer = new HybridBinarizer(source);
var results = reader.decodeMultiple(new BinaryBitmap(binarizer));
if (results != null && results.Length > 0)
{
await Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, () =>
{
foreach (var result in results)
{
if (!readed.Contains(result.Text))
{
readed.Add(result.Text);
Textbox.Text += result.Text + "\n";
}
}
});
}
watch.Stop();
int elapsed = (int)watch.ElapsedMilliseconds;
//run at max 5Hz
await Task.Delay(Math.Max(0, 200 - elapsed));
}
});
}
// Variant of CopyImageUsingMemoryStream avoiding use of pixelBuffer.ToArray(), but
// new SoftwareBitmap() and SoftwareBitmap.CopyFromBuffer have the save duplication cost
internal async static Task CopyImageUsingMemoryStreamAndSoftwareBitmap(IBuffer pixelBuffer, int width, int height)
{
SoftwareBitmap sbmp = new SoftwareBitmap(BitmapPixelFormat.Bgra8, width, height, BitmapAlphaMode.Straight);
sbmp.CopyFromBuffer(pixelBuffer);
ma1 = new InMemoryRandomAccessStream();
BitmapEncoder be = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, ma1);
be.SetSoftwareBitmap(sbmp);
await be.FlushAsync();
var rasr = RandomAccessStreamReference.CreateFromStream(ma1);
ClipboardSetImage(rasr);
}
private async void RunOCR()
{
var ocr = OcrEngine.TryCreateFromLanguage(new Windows.Globalization.Language("en"));
var srcImg = (WriteableBitmap)cropimage.Source;
var ocrImg = new SoftwareBitmap(BitmapPixelFormat.Rgba8, srcImg.PixelWidth, srcImg.PixelHeight);
ocrImg.CopyFromBuffer(srcImg.PixelBuffer);
var ocrResult = await ocr.RecognizeAsync(ocrImg);
string text = "";
foreach (var line in ocrResult.Lines)
{
text += line.Text + "\n";
}
this.ocrResult.Text = text;
}
public static async Task <StorageFile> WriteableBitmapSaveToFile(WriteableBitmap wb, StorageFile file)
{
if (wb != null && file != null)
{
using (var fileStream = await file.OpenAsync(FileAccessMode.ReadWrite))
{
SoftwareBitmap softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, wb.PixelWidth, wb.PixelHeight);
softwareBitmap.CopyFromBuffer(wb.PixelBuffer);
BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, fileStream);
encoder.SetSoftwareBitmap(softwareBitmap);
await encoder.FlushAsync();
}
}
return(file);
}
//若成功拍摄了照片，则要将其转化为模型的输入类型
void FrameConvert2VideoFrame()
{
//Bgra8和Bgra32像素格式应该都是各channel占8Bits，一样的，待验证。
//将HoloLens拍摄获得的PhotoCaptureFrame=》字节数组=》IBuffer=》SoftwareBitmap=》VideoFrame，作为模型的输入。
//待验证经类型转化保存而得的VideoFrame是否正确
IBuffer imageIBuffer = imageBufferArray.AsBuffer();
SoftwareBitmap softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, cameraResolution.width, cameraResolution.height, BitmapAlphaMode.Premultiplied);
softwareBitmap.CopyFromBuffer(imageIBuffer);
//TinyYoloModel的输入是inputImage
inputImage = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
//生成模型输入后，imageBufferArray要重新置空，以进行下次拍摄成功判断
imageBufferArray = null;
imageConvertCount++;
UnityEngine.Debug.Log("PhotoCaptureFrame Convert to VideoFrame Succeed! " + imageConvertCount);
//获取都输入图片后开始预测
EvaluateVideoFrame(inputImage);
}
private async Task <bool> Refresh()
{
bool ret = false;
WriteableBitmap bmp = await this.CamHelper.GetImage();
this.Recognizer.SetFrame(bmp);
if (this.Recognizer.FilterContours())
{
if (this.Recognizer.FindCorners())
{
if (this.Recognizer.IsolateCard())
{
// Successfully framed
WriteableBitmap nameBmp = this.Recognizer.GetNameRegion();
this.NameRegion = nameBmp;
SoftwareBitmap sbmp = new SoftwareBitmap(BitmapPixelFormat.Bgra8, nameBmp.PixelWidth, nameBmp.PixelHeight);
sbmp.CopyFromBuffer(nameBmp.PixelBuffer);
OcrResult result = await this.ocrEngine.RecognizeAsync(sbmp);
this.OCRName = $"'{result.Text}'";
if (result.Text != null && result.Text.Length > 0)
{
CardDB.Card card = CardDB.Instance.Get(result.Text);
this.RecognizedCard = card;
if (this.RecognizedCard != null)
{
ret = true;
this.CardRecognized?.Invoke();
}
}
}
this.TransformedImage = this.Recognizer.GetTransformedCard();
}
this.CornersImage = this.Recognizer.GetCornersDebug();
}
this.ContourImage = this.Recognizer.GetContourDebug();
return(ret);
}
private async void ImageLoaded(object sender, RoutedEventArgs r)
{
using (var stream = await item.path.OpenAsync(FileAccessMode.Read))
{
var bitmapDecoder = await BitmapDecoder.CreateAsync(stream);
var pixelProvider = await bitmapDecoder.GetPixelDataAsync();
bits = pixelProvider.DetachPixelData();
var softwareBitmap = new SoftwareBitmap(
BitmapPixelFormat.Bgra8,
(int)bitmapDecoder.PixelWidth,
(int)bitmapDecoder.PixelHeight,
BitmapAlphaMode.Premultiplied);
softwareBitmap.CopyFromBuffer(bits.AsBuffer());
var softwareBitmapSource = new SoftwareBitmapSource();
await softwareBitmapSource.SetBitmapAsync(softwareBitmap);
img.Source = softwareBitmapSource;
}
}
public async Task <StorageFile> WriteToFile(WriteableBitmap wb)
{
SoftwareBitmap softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, wb.PixelWidth, wb.PixelHeight);
softwareBitmap.CopyFromBuffer(wb.PixelBuffer);
string fileName = Path.GetRandomFileName() + ".png";
StorageFile file = null;
if (softwareBitmap != null)
{
// save image file to cache
file = await ApplicationData.Current.LocalFolder.CreateFileAsync(fileName, CreationCollisionOption.OpenIfExists);
using (IRandomAccessStream stream = await file.OpenAsync(FileAccessMode.ReadWrite))
{
BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, stream);
encoder.SetSoftwareBitmap(softwareBitmap);
await encoder.FlushAsync();
}
}
return(file);
}
public void Resize(ref ImageCanvasDataService imageHolst)
{
int width = imageHolst.Width;
int height = imageHolst.Height;
var softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8,
imageHolst.Image.PixelWidth,
imageHolst.Image.PixelHeight);
imageHolst.Image.CopyTo(softwareBitmap);
byte[] imageBytes = new byte[softwareBitmap.PixelHeight * softwareBitmap.PixelWidth * 4];
softwareBitmap.CopyToBuffer(imageBytes.AsBuffer());
var resourceCreator = CanvasDevice.GetSharedDevice();
var canvasBitmap = CanvasBitmap.CreateFromBytes(
resourceCreator,
imageBytes,
softwareBitmap.PixelWidth,
softwareBitmap.PixelHeight,
Windows.Graphics.DirectX.DirectXPixelFormat.B8G8R8A8UIntNormalized,
96.0f);
var canvasRenderTarget = new CanvasRenderTarget(resourceCreator, width, height, 96);
using (var cds = canvasRenderTarget.CreateDrawingSession())
{
cds.DrawImage(canvasBitmap, canvasRenderTarget.Bounds);
}
var pixelBytes = canvasRenderTarget.GetPixelBytes();
imageHolst.ImageSRC.DecodePixelHeight = height;
imageHolst.ImageSRC.DecodePixelWidth = width;
var scaledSoftwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, width, height);
scaledSoftwareBitmap.CopyFromBuffer(pixelBytes.AsBuffer());
imageHolst.Image = scaledSoftwareBitmap;
}
public async void ReadXml(XmlReader reader)
{
reader.Read();
Title = reader.ReadElementContentAsString("Title", "");
Description = reader.ReadElementContentAsString("Description", "");
string s = reader.ReadElementContentAsString("Status", "");
Status = (JobStatus)Enum.Parse(typeof(JobStatus), s);
if (!reader.IsEmptyElement)
{
InMemoryRandomAccessStream ms = new InMemoryRandomAccessStream();
DataWriter dw = new DataWriter(ms.GetOutputStreamAt(0));
byte[] tempBytes;
int bytesRead = 0;
int totalBytes = 0;
do
{
tempBytes = new byte[1024];
bytesRead = reader.ReadElementContentAsBinHex(tempBytes, 0, 1024);
if (bytesRead > 0)
{
dw.WriteBytes(tempBytes);
totalBytes += bytesRead;
}
} while (bytesRead == 1024);
await dw.StoreAsync();
if (totalBytes > 1)
{
InkStrokeContainer inkCont = new InkStrokeContainer();
await inkCont.LoadAsync(ms);
Strokes = inkCont.GetStrokes().ToList();
}
reader.ReadEndElement();
}
else
{
reader.Read();
}
if (!reader.IsEmptyElement)
{
InMemoryRandomAccessStream ms = new InMemoryRandomAccessStream();
DataWriter dw = new DataWriter(ms.GetOutputStreamAt(0));
byte[] tempBytes;
int bytesRead = 0;
int totalBytesRead = 0;
do
{
tempBytes = new byte[1024];
bytesRead = reader.ReadContentAsBinHex(tempBytes, 0, 1024);
if (bytesRead > 0)
{
dw.WriteBytes(tempBytes);
totalBytesRead += bytesRead;
}
} while (bytesRead == 1024);
await dw.StoreAsync();
if (totalBytesRead > 1)
{
//load bytes as image
byte[] bytes = new byte[ms.Size];
//var dataWriter = new DataWriter(ms);
var dataReader = new DataReader(ms.GetInputStreamAt(0));
await dataReader.LoadAsync((uint)ms.Size);
dataReader.ReadBytes(bytes);
//TODO: this should change based on the resolution you store the photos at
Photo = new SoftwareBitmap(BitmapPixelFormat.Bgra8, 640, 360);
Photo.CopyFromBuffer(bytes.AsBuffer());
}
reader.ReadEndElement();
}
else
{
reader.Read();
}
reader.Skip();
}
private async Task MachineLearningTask()
{
var ModelInput = new NumberModelInput();
var ModelGen = new NumberModel();
var ModelOutput = new NumberModelOutput();
try
{
StorageFile modelFile = await ApplicationData.Current.LocalFolder.GetFileAsync(onnxName);
ModelGen = await NumberModel.CreateNumberModel(modelFile);
}
catch (Exception e)
{
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
TextData.text = e.ToString();
}, true);
}
while (true)
{
if (bytes != null)
{
// Unityのカメラは上下反転しているので入れ替え処理
var buf = new byte[bytes.Length];
for (int i = 0; i < TexHeight; i++)
{
for (int j = 0; j < TexWidth; j++)
{
buf[(TexWidth * (TexHeight - 1 - i) + j) * 4 + 0] = bytes[(TexWidth * i + j) * 4 + 0];
buf[(TexWidth * (TexHeight - 1 - i) + j) * 4 + 1] = bytes[(TexWidth * i + j) * 4 + 1];
buf[(TexWidth * (TexHeight - 1 - i) + j) * 4 + 2] = bytes[(TexWidth * i + j) * 4 + 2];
buf[(TexWidth * (TexHeight - 1 - i) + j) * 4 + 3] = bytes[(TexWidth * i + j) * 4 + 3];
}
}
try
{
var softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Rgba8, TexWidth, TexHeight, BitmapAlphaMode.Premultiplied);
softwareBitmap.CopyFromBuffer(buf.AsBuffer());
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
ModelInput.data = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
ModelOutput = await ModelGen.EvaluateAsync(ModelInput);
float maxProb = 0;
string maxIndexName = "";
foreach (var item in ModelOutput.loss)
{
if (item.Value > maxProb)
{
maxIndexName = item.Key;
maxProb = item.Value;
}
}
softwareBitmap.Dispose();
bytes = null;
buf = null;
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
TextData.text = maxIndexName + ":" + maxProb;
}, true);
}
catch (Exception e)
{
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
TextData.text = e.ToString();
}, true);
}
}
}
}
protected override async void OnRun(IBackgroundTaskInstance taskInstance)
{
var deferral = taskInstance.GetDeferral();
// Create instance of the control that contains the layout of the Live Tile
MediumTileControl control = new MediumTileControl();
control.Width = TileWidth;
control.Height = TileHeight;
// If we have received a message in the parameters, overwrite the default "Hello, Live Tile!" one
var triggerDetails = taskInstance.TriggerDetails as ApplicationTriggerDetails;
if (triggerDetails != null)
{
object tileMessage = null;
if (triggerDetails.Arguments.TryGetValue("Message", out tileMessage))
{
if (tileMessage is string)
{
control.Message = (string)tileMessage;
}
}
}
// Render the tile control to a RenderTargetBitmap
RenderTargetBitmap bitmap = new RenderTargetBitmap();
await bitmap.RenderAsync(control, TileWidth, TileHeight);
// Now we are going to save it to a PNG file, so create/open it on local storage
var file = await ApplicationData.Current.LocalFolder.CreateFileAsync(TileImageFilename, CreationCollisionOption.ReplaceExisting);
using (var stream = await file.OpenAsync(FileAccessMode.ReadWrite))
{
// Create a BitmapEncoder for encoding to PNG
BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, stream);
// Create a SoftwareBitmap from the RenderTargetBitmap, as it will be easier to save to disk
using (var softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Bgra8, TileWidth, TileHeight, BitmapAlphaMode.Premultiplied))
{
// Copy bitmap data
softwareBitmap.CopyFromBuffer(await bitmap.GetPixelsAsync());
// Encode and save to file
encoder.SetSoftwareBitmap(softwareBitmap);
await encoder.FlushAsync();
}
}
// Use the NotificationsExtensions library to easily configure a tile update
TileContent mediumTileContent = new TileContent()
{
Visual = new TileVisual()
{
TileMedium = new TileBinding()
{
Content = new TileBindingContentAdaptive()
{
BackgroundImage = new TileBackgroundImage()
{
Overlay = 0,
Source = new TileImageSource("ms-appdata:///local/" + TileImageFilename),
}
}
}
}
};
// Clean previous update from Live Tile and update with the new parameters
var tileUpdater = TileUpdateManager.CreateTileUpdaterForApplication();
tileUpdater.Clear();
tileUpdater.Update(new TileNotification(mediumTileContent.GetXml()));
deferral.Complete();
}
async public Task <RandomAccessStreamReference> GetImage()
{
// 1. Příprava
CanvasDevice device = CanvasDevice.GetSharedDevice();
// 2. Získáme pozadí jako bitmapu
RenderTargetBitmap renderTargetBitmap = new RenderTargetBitmap();
await renderTargetBitmap.RenderAsync(sourceGrid, (int)sourceGrid.ActualWidth, (int)sourceGrid.ActualHeight);
// 3. Zafixujeme rozměry bitmapy v pixelech
var bitmapSizeAt96Dpi = new Size(renderTargetBitmap.PixelWidth, renderTargetBitmap.PixelHeight);
// 4. Získáme pixely
IBuffer pixelBuffer = await renderTargetBitmap.GetPixelsAsync();
byte[] pixels = pixelBuffer.ToArray();
// 5. Začneme renderovat při 96 DPI
using (CanvasRenderTarget renderTarget = new CanvasRenderTarget(device, (int)sourceGrid.ActualWidth, (int)sourceGrid.ActualHeight, 96.0f))
{
using (var drawingSession = renderTarget.CreateDrawingSession())
{
using (var win2dRenderedBitmap = CanvasBitmap.CreateFromBytes(device, pixels,
(int)bitmapSizeAt96Dpi.Width, (int)bitmapSizeAt96Dpi.Height,
Windows.Graphics.DirectX.DirectXPixelFormat.B8G8R8A8UIntNormalized,
96.0f))
{
drawingSession.DrawImage(win2dRenderedBitmap,
new Rect(0, 0, renderTarget.SizeInPixels.Width, renderTarget.SizeInPixels.Height),
new Rect(0, 0, bitmapSizeAt96Dpi.Width, bitmapSizeAt96Dpi.Height));
}
// 6. Přidáme ink
drawingSession.Units = CanvasUnits.Pixels;
drawingSession.DrawInk(inkCanvas.InkPresenter.StrokeContainer.GetStrokes());
}
var outputBitmap = new SoftwareBitmap(
BitmapPixelFormat.Bgra8,
(int)renderTarget.SizeInPixels.Width,
(int)renderTarget.SizeInPixels.Height,
BitmapAlphaMode.Premultiplied);
outputBitmap.CopyFromBuffer(renderTarget.GetPixelBytes().AsBuffer());
// 7. Vykreslíme do bufferu
var stream = new InMemoryRandomAccessStream();
var encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.BmpEncoderId, stream);
encoder.SetPixelData(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Straight,
(uint)renderTarget.SizeInPixels.Width, (uint)renderTarget.SizeInPixels.Height,
96.0f, 96.0f,
renderTarget.GetPixelBytes()
);
await encoder.FlushAsync();
stream.Seek(0);
return(RandomAccessStreamReference.CreateFromStream(stream));
}
}
/// <summary>
/// Every x milliseconds, capture a frame from the webcam and process for OCR.
/// </summary>
/// <param name="threshold"></param>
/// <returns></returns>
private async Task ProcessFrameOcr(double threshold)
{
//Get information about the preview.
var previewProperties = _mediaCapture.VideoDeviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties;
int videoFrameWidth = (int)previewProperties.Width;
int videoFrameHeight = (int)previewProperties.Height;
if (!externalCamera && (displayInformation.CurrentOrientation == DisplayOrientations.Portrait || displayInformation.CurrentOrientation == DisplayOrientations.PortraitFlipped))
{
videoFrameWidth = (int)previewProperties.Height;
videoFrameHeight = (int)previewProperties.Width;
}
// Create the video frame to request a SoftwareBitmap preview frame.
var videoFrame = new VideoFrame(BitmapPixelFormat.Bgra8, videoFrameWidth, videoFrameHeight);
// Capture the preview frame.
using (var currentFrame = await _mediaCapture.GetPreviewFrameAsync(videoFrame))
{
// Collect the resulting frame.
SoftwareBitmap bitmap = currentFrame.SoftwareBitmap;
WriteableBitmap processedBitmap;
SoftwareBitmap bitmapToOcr;
int oneThirdHeight = bitmap.PixelHeight / 3;
processedBitmap = new WriteableBitmap(bitmap.PixelWidth, bitmap.PixelHeight);
bitmap.CopyToBuffer(processedBitmap.PixelBuffer);
processedBitmap = await Helpers.ImageProcessor.ApplyGaussianBlur(processedBitmap, 4);
if (IsImageEffectsOn)
{
processedBitmap = await Helpers.ImageProcessor.AdjustCurvesEffect(processedBitmap);
processedBitmap = await Helpers.ImageProcessor.ApplyStampThreshold(processedBitmap, threshold);
Log.Information($"IMG - threshold @ {threshold}, gaussian at 4 applied before OCR.");
}
//set the preview image source - if doing image preprocessing and effects
if (_isOcrStarted)
{
PreviewImageSource = processedBitmap;
PreviewImageIsVisible = true;
CaptureElementIsVisible = false;
}
if (IsOneThirdCapture)
{
Rect rectOneThird = new Rect(0, 0, bitmap.PixelWidth, bitmap.PixelHeight / 3);
WriteableBitmap cropped = Helpers.ImageProcessor.CropToRectangle(processedBitmap, rectOneThird);
bitmapToOcr = new SoftwareBitmap(BitmapPixelFormat.Bgra8, cropped.PixelWidth, cropped.PixelHeight);
bitmapToOcr.CopyFromBuffer(cropped.PixelBuffer);
}
else
{
bitmapToOcr = new SoftwareBitmap(BitmapPixelFormat.Bgra8, processedBitmap.PixelWidth, processedBitmap.PixelHeight);
bitmapToOcr.CopyFromBuffer(processedBitmap.PixelBuffer);
}
Tuple <string, int> result = null;
result = await _ocrDetectionService.PerformOcr(bitmapToOcr, _accuracyThreshold, _isOneThirdCapture, oneThirdHeight);
if (result != null && result.Item1 != null && result.Item1 != "")
{
//add to internal list keep track of spoken sentences - this is useful if an empty result is returned between two similar Ocr results (can happen if contrast is really bad for a few milliseconds)
if (!SentenceHasBeenSpoken(result.Item1))
{
_trackSpokenSentences.Add(result.Item1);
_ttsSynthesizer.AddUtteranceToSynthesize(result.Item1);
await SpeechSynthiserCall();
Log.Debug($"OCR result spoken and assigning to Textblox onscreen (Conf: {result.Item2}) - {result.Item1}");
OcrResultText = result.Item1;
}
StatusMessage = "";
StatusBackground = new SolidColorBrush(Windows.UI.Colors.Transparent);
}
}
}
private async Task DisplayAllFiles()
{
Debug.WriteLine("-- GETTING FILES");
StorageFolder appInstalledFolder = Windows.ApplicationModel.Package.Current.InstalledLocation;
StorageFolder dataFolder = await appInstalledFolder.GetFolderAsync(GlobalData.dataFolder);
IReadOnlyList <StorageFile> fileList = await dataFolder.GetFilesAsync();
this.ImageHolder.Opacity = 0;
this.ImageHolder.Width = 1500;
this.ImageHolder.Height = 1500;
this.ImageHolder.ColumnDefinitions.Add(new ColumnDefinition()
{
Width = GridLength.Auto
});
this.ImageHolder.ColumnDefinitions.Add(new ColumnDefinition()
{
Width = new GridLength()
});
this.ImageHolder.ColumnDefinitions.Add(new ColumnDefinition()
{
Width = new GridLength()
});
this.ImageHolder.ColumnDefinitions.Add(new ColumnDefinition()
{
Width = new GridLength()
});
this.ImageHolder.ColumnDefinitions.Add(new ColumnDefinition()
{
Width = new GridLength()
});
this.ImageHolder.ColumnDefinitions.Add(new ColumnDefinition()
{
Width = new GridLength()
});
this.ImageHolder.RowDefinitions.Add(new RowDefinition()
{
Height = new GridLength()
});
Button button = new Button();
button.Name = "IDC_Classify";
button.Content = "Classify All Images";
button.Click += IDC_Classify_Click;
button.Width = 250;
button.Height = 75;
button.Foreground = new SolidColorBrush(Windows.UI.Colors.White);
button.SetValue(Grid.ColumnProperty, 6);
button.SetValue(Grid.RowProperty, 0);
this.ImageHolder.Children.Add(button);
this.ImageHolder.RowDefinitions.Add(new RowDefinition()
{
Height = new GridLength()
});
var result = new ObservableCollection <BitmapImage>();
var i = 0;
var row = 1;
var column = 0;
foreach (StorageFile file in fileList)
{
Debug.WriteLine(file.Name);
using (var stream = await file.OpenAsync(FileAccessMode.Read))
{
var bitmapDecoder = await BitmapDecoder.CreateAsync(stream);
var pixelProvider = await bitmapDecoder.GetPixelDataAsync();
var bits = pixelProvider.DetachPixelData();
var softwareBitmap = new SoftwareBitmap(
BitmapPixelFormat.Bgra8,
(int)bitmapDecoder.PixelWidth,
(int)bitmapDecoder.PixelHeight,
BitmapAlphaMode.Premultiplied);
softwareBitmap.CopyFromBuffer(bits.AsBuffer());
var softwareBitmapSource = new SoftwareBitmapSource();
await softwareBitmapSource.SetBitmapAsync(softwareBitmap);
var source = new SoftwareBitmapSource();
await source.SetBitmapAsync(softwareBitmap);
if (i != 0 && i % 6 == 0)
{
Debug.WriteLine("-- New Image Row " + row);
this.ImageHolder.RowDefinitions.Add(
new RowDefinition()
{
Height = GridLength.Auto
});
column = 0;
row++;
}
Image image = new Image();
image.Source = softwareBitmapSource;
image.Name = file.Name;
image.Width = 250;
image.Height = 250;
image.SetValue(Grid.ColumnProperty, column);
image.SetValue(Grid.RowProperty, row);
this.ImageHolder.Children.Add(image);
column++;
i++;
}
}
this.ImageHolder.Opacity = 1;
}
private async Task ObjectDetectation()
{
while (true)
{
if (bytes != null)
{
// UnityのWebカメラは上下反転しているので入れ替え処理
var buf = new byte[bytes.Length];
for (int i = 0; i < 504; i++)
{
for (int j = 0; j < 896; j++)
{
buf[(896 * (504 - 1 - i) + j) * 4 + 0] = bytes[(896 * i + j) * 4 + 0];
buf[(896 * (504 - 1 - i) + j) * 4 + 1] = bytes[(896 * i + j) * 4 + 1];
buf[(896 * (504 - 1 - i) + j) * 4 + 2] = bytes[(896 * i + j) * 4 + 2];
buf[(896 * (504 - 1 - i) + j) * 4 + 3] = bytes[(896 * i + j) * 4 + 3];
}
}
// 入力画像から物体認識を行う
try
{
SoftwareBitmap softwareBitmap;
softwareBitmap = new SoftwareBitmap(BitmapPixelFormat.Rgba8, 896, 504);
softwareBitmap.CopyFromBuffer(buf.AsBuffer());
bytes = null;
buf = null;
softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
VideoFrame inputFrame = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
// WindowsMLに入力，出力形式を設定する
LearningModelBindingPreview binding = new LearningModelBindingPreview(_model as LearningModelPreview);
binding.Bind(_inputImageDescription.Name, inputFrame);
binding.Bind(_outputTensorDescription.Name, _outputVariableList);
// Process the frame with the model
LearningModelEvaluationResultPreview results = await _model.EvaluateAsync(binding, "test");
List <float> resultProbabilities = results.Outputs[_outputTensorDescription.Name] as List <float>;
// 認識結果から適合率の高い上位3位までを選択
List <float> topProbabilities = new List <float>()
{
0.0f, 0.0f, 0.0f
};
List <int> topProbabilityLabelIndexes = new List <int>()
{
0, 0, 0
};
for (int i = 0; i < resultProbabilities.Count(); i++)
{
for (int j = 0; j < 3; j++)
{
if (resultProbabilities[i] > topProbabilities[j])
{
topProbabilityLabelIndexes[j] = i;
topProbabilities[j] = resultProbabilities[i];
break;
}
}
}
// 結果を出力する
string message = "Predominant objects detected are:";
for (int i = 0; i < 3; i++)
{
message += $"\n{ _labels[topProbabilityLabelIndexes[i]]} with confidence of { topProbabilities[i]}";
}
softwareBitmap.Dispose();
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
text.text = message;
}, true);
}
catch (Exception ex)
{
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
text.text = ex.ToString();
}, true);
}
}
}
}