public async Task<byte[]> ConvertToGrayScale(byte[] imageBytes, int height, int width)
{
using (InMemoryRandomAccessStream rasStream = new InMemoryRandomAccessStream())
{
await rasStream.WriteAsync(imageBytes.AsBuffer());
var decoder = await BitmapDecoder.CreateAsync(rasStream);
var pixelData = await decoder.GetPixelDataAsync();
var pixels = pixelData.DetachPixelData();
if (_filter == null)
_filter = new ImageFilter();
await _filter.ToGrayScale(pixels.AsBuffer());
BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, rasStream);
encoder.SetPixelData(decoder.BitmapPixelFormat, BitmapAlphaMode.Ignore, (uint)width, (uint)height, decoder.DpiX, decoder.DpiY, pixels);
await encoder.FlushAsync();
using (BinaryReader br = new BinaryReader(rasStream.AsStreamForRead()))
{
rasStream.Seek(0);
return br.ReadBytes((int)rasStream.AsStreamForRead().Length);
}
}
}
public static async Task LoadImages()
{
for (int i = 0; i < NUM_ICON_SIZES; i++)
{
for (int j = 0; j < NUM_ICON_TYPES; j++)
{
string postfix;
if (j >= 1 && j <= 8)
postfix = "BusDirection" + ((StopDirection)j).ToString();
else if (j == 0)
postfix = "BusBase";
else
postfix = j == 9 ? "BusAlert" : "BusClosed";
postfix += (i == 0 ? "20" : "40");
var sprite = new Sprite() { ImageUri = new Uri($"ms-appx:///Assets/Icons/{postfix}.png") };
await sprite.Load();
sprite.Unlock();
//var buffer1 = new SpriteBitmapStream(sprite).GetFullBuffer();
//using (MemoryStream stream1 = new MemoryStream(buffer1, true))
//{
// BusIconStreams[i * NUM_ICON_TYPES + j] = stream1.AsRandomAccessStream();
//}
//var bitmap = await WriteableBitmapExtensions.FromContent(null, new Uri($"ms-appx:///Assets/Icons/{postfix}.png"));
InMemoryRandomAccessStream stream = new InMemoryRandomAccessStream();
await sprite.Bitmap.ToStream(stream, BitmapEncoder.BmpEncoderId);
BusIconStreams[i * NUM_ICON_TYPES + j] = stream;
byte[] buffer = new byte[stream.Size];
stream.AsStreamForRead().Read(buffer, 0, buffer.Length);
for (int k = 0; k < buffer.Length; k++)
{
//if (buffer[k] != buffer1[k])
//{
// //System.Diagnostics.Debug.WriteLine($"{buffer[k]} vs {buffer1[k]}");
//}
}
}
}
}
public Stream CompressBitmap(byte[] data, int w, int h)
{
var memStream = new InMemoryRandomAccessStream();
Encode(data, (uint)w, (uint)h, memStream);
var stream = memStream.AsStreamForRead();
Logger($"Log Encode stream for bugfix:{memStream.Size} => {stream.Length} bytes");
return stream;
}
/// <summary>
/// This is an infinite loop which
/// takes a picture with the attached camera,
/// displays it,
/// sends it for recognition to the Microsoft Project Oxford Face API,
/// displays recognition results overlaid on the picture,
/// waits for 5 seconds to allow the result to be examined,
/// starts over.
/// </summary>
private async Task RunControlLoopAsync()
{
while (true)
{
// Take camera picture
await UpdateStatusAsync("Taking still picture...");
// TODO focus if possible
//await mediaCapture.VideoDeviceController.FocusControl.FocusAsync();
FaceResultsGrid.Children.Clear();
CountdownProgressBar.Value = 100;
CameraFlashStoryboard.Begin();
using (var stream = new InMemoryRandomAccessStream())
{
var imageEncodingProperties = ImageEncodingProperties.CreatePng();
imageEncodingProperties.Width = 320;
imageEncodingProperties.Height = 200;
await mediaCapture.CapturePhotoToStreamAsync(imageEncodingProperties, stream);
// Display camera picture
await UpdateStatusAsync("Displaying sample picture...");
stream.Seek(0);
var bitmapImage = new BitmapImage();
await bitmapImage.SetSourceAsync(stream);
ResultImage.Source = bitmapImage;
// Send picture for recognition
// We need to encode the raw image as a JPEG to make sure the service can recognize it.
await UpdateStatusAsync("Uploading picture to Microsoft Project Oxford Face API...");
stream.Seek(0);
var recognizedFaces = await GetFaces(stream.AsStreamForRead());
// Display recognition results
// Wait a few seconds seconds to give viewers a chance to appreciate all we've done
await UpdateStatusAsync($"{recognizedFaces.Count()} face(s) found by Microsoft 'Project Oxford' Face API");
// The face rectangles received from Face API are measured in pixels of the raw image.
// We need to calculate the extra scaling and displacement that results from the raw image
// being displayed in a larger container.
// We use the FaceResultsGrid as a basis for the calculation, because the ResultImage control's ActualHeight and ActualWidth
// properties have the same aspect ratio as the image, and not the aspect ratio of the screen.
double widthScaleFactor = FaceResultsGrid.ActualWidth / bitmapImage.PixelWidth;
double heightScaleFactor = FaceResultsGrid.ActualHeight / bitmapImage.PixelHeight;
double scaleFactor = Math.Min(widthScaleFactor, heightScaleFactor);
bool isTheBlackSpaceOnTheLeft = widthScaleFactor > heightScaleFactor;
double extraLeftNeeded = 0;
double extraTopNeeded = 0;
if (isTheBlackSpaceOnTheLeft) extraLeftNeeded = (FaceResultsGrid.ActualWidth - scaleFactor * bitmapImage.PixelWidth) / 2;
else extraTopNeeded = (FaceResultsGrid.ActualHeight - scaleFactor * bitmapImage.PixelHeight) / 2;
foreach (var face in recognizedFaces)
{
var faceOutlineRectangleLeft = extraLeftNeeded + scaleFactor * face.FaceRectangle.Left;
var faceOutlineRectangleTop = extraTopNeeded + scaleFactor * face.FaceRectangle.Top;
var faceOutlineRectangleHeight = scaleFactor * face.FaceRectangle.Height;
var faceOutlineRectangleWidth = scaleFactor * face.FaceRectangle.Width;
Rectangle faceOutlineRectangle = new Rectangle();
faceOutlineRectangle.Stroke = new SolidColorBrush(Colors.Black);
faceOutlineRectangle.StrokeThickness = 3;
faceOutlineRectangle.HorizontalAlignment = HorizontalAlignment.Left;
faceOutlineRectangle.VerticalAlignment = VerticalAlignment.Top;
faceOutlineRectangle.Margin = new Thickness(faceOutlineRectangleLeft, faceOutlineRectangleTop, 0, 0);
faceOutlineRectangle.Height = faceOutlineRectangleHeight;
faceOutlineRectangle.Width = faceOutlineRectangleWidth;
FaceResultsGrid.Children.Add(faceOutlineRectangle);
TextBlock faceInfoTextBlock = new TextBlock();
faceInfoTextBlock.Foreground = new SolidColorBrush(Colors.White);
faceInfoTextBlock.FontSize = 30;
faceInfoTextBlock.Text = $"{face.Attributes.Gender}, {face.Attributes.Age}";
Border faceInfoBorder = new Border();
faceInfoBorder.Background = new SolidColorBrush(Colors.Black);
faceInfoBorder.Padding = new Thickness(5);
faceInfoBorder.Child = faceInfoTextBlock;
faceInfoBorder.HorizontalAlignment = HorizontalAlignment.Left;
faceInfoBorder.VerticalAlignment = VerticalAlignment.Top;
faceInfoBorder.Margin = new Thickness(faceOutlineRectangleLeft, faceOutlineRectangleTop - 50, 0, 0);
FaceResultsGrid.Children.Add(faceInfoBorder);
TextBlock carInfoTextBlock = new TextBlock();
carInfoTextBlock.Foreground = new SolidColorBrush(Colors.White);
carInfoTextBlock.FontSize = 30;
carInfoTextBlock.Text = GetCarRecommendation(face.Attributes.Gender, (int)face.Attributes.Age);
Border carInfoBorder = new Border();
carInfoBorder.Background = new SolidColorBrush(Colors.Black);
carInfoBorder.Padding = new Thickness(5);
carInfoBorder.Child = carInfoTextBlock;
carInfoBorder.HorizontalAlignment = HorizontalAlignment.Left;
carInfoBorder.VerticalAlignment = VerticalAlignment.Top;
carInfoBorder.Margin = new Thickness(faceOutlineRectangleLeft, faceOutlineRectangleTop + faceOutlineRectangleHeight, 0, 0);
FaceResultsGrid.Children.Add(carInfoBorder);
}
}
CountdownStoryboard.Begin();
await Task.Delay(ControlLoopDelayMilliseconds);
}
}
private async void AnalyzeButton_Click(object sender, RoutedEventArgs e)
{
if (processingImage)
{
// Ignore button presses while processing the image
return;
}
if (inCaptureState)
{
processingImage = true;
inCaptureState = false;
// Make the 'Processing...' label visible
canvasControl.Visibility = Visibility.Visible;
AnalyzeButton.Content = "...";
canvasControl.Invalidate();
var originalPhoto = new InMemoryRandomAccessStream();
var reencodedPhoto = new InMemoryRandomAccessStream();
await mediaCapture.CapturePhotoToStreamAsync(ImageEncodingProperties.CreateJpeg(), originalPhoto);
await originalPhoto.FlushAsync();
originalPhoto.Seek(0);
captureElement.Visibility = Visibility.Collapsed;
// Store the captured photo as a Win2D type for later use
photoCanvasBitmap = await CanvasBitmap.LoadAsync(canvasControl, originalPhoto);
// Send the photo to Project Oxford to detect the faces
lastCapturedFaces = await faceServiceClient.DetectAsync(originalPhoto.AsStreamForRead(), true, true, true, false);
// Force the canvasControl to be redrawn now that the photo is available
canvasControl.Invalidate();
processingImage = false;
AnalyzeButton.Content = "Restart";
}
else
{
canvasControl.Visibility = Visibility.Collapsed;
captureElement.Visibility = Visibility.Visible;
AnalyzeButton.Content = "Capture Photo";
photoCanvasBitmap = null;
canvasControl.Invalidate();
inCaptureState = true;
}
}
/// <summary>
/// Uploads a photo as a WriteableBitmap. This methods converts the given bitmap to a PNG file before sending it to the server.
/// </summary>
/// <param name="uri"></param>
/// <param name="bmp"></param>
/// <returns></returns>
public static async Task UploadImage(string uri, WriteableBitmap bmp)
{
InMemoryRandomAccessStream memoryStream = new InMemoryRandomAccessStream();
BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, memoryStream);
encoder.SetPixelData(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied, (uint)bmp.PixelWidth, (uint)bmp.PixelHeight, 96, 96, bmp.PixelBuffer.ToArray());
await encoder.FlushAsync();
Stream stream = memoryStream.AsStreamForRead();
byte[] pngBuffer = new byte[stream.Length];
stream.Read(pngBuffer, 0, pngBuffer.Length);
await UploadImage(uri, pngBuffer);
}
/// <summary>
/// Encodes the specified bitmap data and outputs it to the specified
/// <c>BinaryWriter</c>. Bitmap data should be in BGRA format.
/// For internal use only.
/// </summary>
public async Task EncodeAsync(byte[] bytes, BinaryWriter writer)
{
#if NETFX_CORE
using (var jpegStream = new InMemoryRandomAccessStream())
{
var propertySet = new BitmapPropertySet();
var qualityValue = new BitmapTypedValue(this.JpegQuality / 100.0, PropertyType.Single);
propertySet.Add("ImageQuality", qualityValue);
var encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, jpegStream, propertySet);
if (this.Width != this.OutputWidth)
{
encoder.BitmapTransform.ScaledWidth = (uint)this.OutputWidth;
encoder.BitmapTransform.ScaledHeight = (uint)this.OutputHeight;
}
encoder.SetPixelData(
BitmapPixelFormat.Bgra8,
BitmapAlphaMode.Straight,
(uint)this.Width,
(uint)this.Height,
96,
96,
bytes);
await encoder.FlushAsync();
if (writer.BaseStream == null || writer.BaseStream.CanWrite == false)
return;
// Header
writer.Write(this.OutputWidth);
writer.Write(this.OutputHeight);
writer.Write((int)jpegStream.Size);
// Data
jpegStream.AsStreamForRead().CopyTo(writer.BaseStream);
}
#else
await Task.Run(() =>
{
var format = PixelFormats.Bgra32;
int stride = (int)this.Width * format.BitsPerPixel / 8;
var bmp = BitmapSource.Create(
this.Width,
this.Height,
96.0,
96.0,
format,
null,
bytes,
stride);
BitmapFrame frame;
if (this.Width != this.OutputWidth || this.Height != this.OutputHeight)
{
var transform = new ScaleTransform((double)this.OutputHeight / this.Height, (double)this.OutputHeight / this.Height);
var scaledbmp = new TransformedBitmap(bmp, transform);
frame = BitmapFrame.Create(scaledbmp);
}
else
{
frame = BitmapFrame.Create(bmp);
}
var encoder = new JpegBitmapEncoder()
{
QualityLevel = this.JpegQuality
};
encoder.Frames.Add(frame);
using (var jpegStream = new MemoryStream())
{
encoder.Save(jpegStream);
if (writer.BaseStream == null || writer.BaseStream.CanWrite == false)
return;
// Header
writer.Write(this.OutputWidth);
writer.Write(this.OutputHeight);
writer.Write((int)jpegStream.Length);
// Data
jpegStream.Position = 0;
jpegStream.CopyTo(writer.BaseStream);
}
});
#endif
}
async Task<string> CaptureAndAnalyze(bool readText = false)
{
var imgFormat = ImageEncodingProperties.CreateJpeg();
//NOTE: this is how you can save a frame to the CameraRoll folder:
//var file = await KnownFolders.CameraRoll.CreateFileAsync($"MCS_Photo{DateTime.Now:HH-mm-ss}.jpg", CreationCollisionOption.GenerateUniqueName);
//await mediaCapture.CapturePhotoToStorageFileAsync(imgFormat, file);
//var stream = await file.OpenStreamForReadAsync();
// Capture a frame and put it to MemoryStream
var memoryStream = new MemoryStream();
using (var ras = new InMemoryRandomAccessStream())
{
await mediaCapture.CapturePhotoToStreamAsync(imgFormat, ras);
ras.Seek(0);
using (var stream = ras.AsStreamForRead())
stream.CopyTo(memoryStream);
}
var imageBytes = memoryStream.ToArray();
memoryStream.Position = 0;
if (withPreview)
{
InvokeOnMain(() =>
{
var image = new Image();
image.Load(new Urho.MemoryBuffer(imageBytes));
Node child = Scene.CreateChild();
child.Position = LeftCamera.Node.WorldPosition + LeftCamera.Node.WorldDirection * 2f;
child.LookAt(LeftCamera.Node.WorldPosition, Vector3.Up, TransformSpace.World);
child.Scale = new Vector3(1f, image.Height / (float)image.Width, 0.1f) / 10;
var texture = new Texture2D();
texture.SetData(image, true);
var material = new Material();
material.SetTechnique(0, CoreAssets.Techniques.Diff, 0, 0);
material.SetTexture(TextureUnit.Diffuse, texture);
var box = child.CreateComponent<Box>();
box.SetMaterial(material);
child.RunActions(new EaseBounceOut(new ScaleBy(1f, 5)));
});
}
try
{
var client = new VisionServiceClient(VisionApiKey);
if (readText)
{
var ocrResult = await client.RecognizeTextAsync(memoryStream, detectOrientation: false);
var words = ocrResult.Regions.SelectMany(region => region.Lines).SelectMany(line => line.Words).Select(word => word.Text);
return "it says: " + string.Join(" ", words);
}
else
{
// just describe the picture, you can also use cleint.AnalyzeImageAsync method to get more info
var result = await client.DescribeAsync(memoryStream);
return result?.Description?.Captions?.FirstOrDefault()?.Text;
}
}
catch (ClientException exc)
{
return exc?.Error?.Message ?? "Failed";
}
catch (Exception exc)
{
return "Failed";
}
}
private async Task ImageUpdate(bool isCommand)
{
DateTime currentTime = DateTime.UtcNow;
// Just incase - stop code being called while photo already in progress
if (this.cameraBusy)
{
return;
}
this.cameraBusy = true;
this.displayGpioPin.Write(GpioPinValue.High);
// Check that enough time has passed for picture to be taken
if ((currentTime - this.imageLastCapturedAtUtc) < this.debounceTimeout)
{
this.displayOffTimer.Change(this.timerPeriodDetectIlluminated, this.timerPeriodInfinite);
return;
}
this.imageLastCapturedAtUtc = currentTime;
try
{
ImagePrediction imagePrediction;
using (Windows.Storage.Streams.InMemoryRandomAccessStream captureStream = new Windows.Storage.Streams.InMemoryRandomAccessStream())
{
this.mediaCapture.CapturePhotoToStreamAsync(ImageEncodingProperties.CreateJpeg(), captureStream).AsTask().Wait();
captureStream.FlushAsync().AsTask().Wait();
captureStream.Seek(0);
IStorageFile photoFile = await KnownFolders.PicturesLibrary.CreateFileAsync(ImageFilename, CreationCollisionOption.ReplaceExisting);
ImageEncodingProperties imageProperties = ImageEncodingProperties.CreateJpeg();
await this.mediaCapture.CapturePhotoToStorageFileAsync(imageProperties, photoFile);
switch (modelType)
{
case ModelType.Classification:
imagePrediction = await this.customVisionClient.ClassifyImageAsync(this.projectId, this.modelPublishedName, captureStream.AsStreamForRead());
break;
case ModelType.Detection:
imagePrediction = await this.customVisionClient.DetectImageAsync(this.projectId, this.modelPublishedName, captureStream.AsStreamForRead());
break;
default:
throw new ArgumentException("ModelType Invalid");
}
Debug.WriteLine($"Prediction count {imagePrediction.Predictions.Count}");
}
JObject telemetryDataPoint = new JObject();
LoggingFields imageInformation = new LoggingFields();
imageInformation.AddDateTime("TakenAtUTC", currentTime);
imageInformation.AddBoolean("IsCommand", isCommand);
imageInformation.AddDouble("Probability threshold", probabilityThreshold);
imageInformation.AddInt32("Predictions", imagePrediction.Predictions.Count);
// Display and log the results of the prediction
foreach (var prediction in imagePrediction.Predictions)
{
Debug.WriteLine($" Tag:{prediction.TagName} {prediction.Probability}");
imageInformation.AddDouble($"Tag:{prediction.TagName}", prediction.Probability);
}
// Post process the predictions based on the type of model
switch (modelType)
{
case ModelType.Classification:
// Use only the tags above the specified minimum probability
foreach (var prediction in imagePrediction.Predictions)
{
if (prediction.Probability >= probabilityThreshold)
{
// Display and log the individual tag probabilities
Debug.WriteLine($" Tag valid:{prediction.TagName} {prediction.Probability:0.00}");
imageInformation.AddDouble($"Tag valid:{prediction.TagName}", prediction.Probability);
telemetryDataPoint.Add(prediction.TagName, prediction.Probability);
}
}
break;
case ModelType.Detection:
// Group the tags to get the count, include only the predictions above the specified minimum probability
var groupedPredictions = from prediction in imagePrediction.Predictions
where prediction.Probability >= probabilityThreshold
group prediction by new { prediction.TagName }
into newGroup
select new
{
TagName = newGroup.Key.TagName,
Count = newGroup.Count(),
};
// Display and log the agregated predictions
foreach (var prediction in groupedPredictions)
{
Debug.WriteLine($" Tag valid:{prediction.TagName} {prediction.Count}");
imageInformation.AddInt32($"Tag valid:{prediction.TagName}", prediction.Count);
telemetryDataPoint.Add(prediction.TagName, prediction.Count);
}
break;
default:
throw new ArgumentException("ModelType Invalid");
}
this.logging.LogEvent("Captured image processed by Cognitive Services", imageInformation);
try
{
using (Message message = new Message(Encoding.ASCII.GetBytes(JsonConvert.SerializeObject(telemetryDataPoint))))
{
Debug.WriteLine(" {0:HH:mm:ss} AzureIoTHubClient SendEventAsync start", DateTime.UtcNow);
await this.azureIoTHubClient.SendEventAsync(message);
Debug.WriteLine(" {0:HH:mm:ss} AzureIoTHubClient SendEventAsync finish", DateTime.UtcNow);
}
this.logging.LogEvent("SendEventAsync payload", imageInformation, LoggingLevel.Information);
}
catch (Exception ex)
{
imageInformation.AddString("Exception", ex.ToString());
this.logging.LogEvent("SendEventAsync payload", imageInformation, LoggingLevel.Error);
}
}
catch (Exception ex)
{
this.logging.LogMessage("Camera photo or save failed " + ex.Message, LoggingLevel.Error);
}
finally
{
this.displayGpioPin.Write(GpioPinValue.Low);
this.cameraBusy = false;
}
}
private async Task ImageUpdate(bool isCommand)
{
DateTime currentTime = DateTime.UtcNow;
// Just incase - stop code being called while photo already in progress
if (this.cameraBusy)
{
return;
}
this.cameraBusy = true;
try
{
using (Windows.Storage.Streams.InMemoryRandomAccessStream captureStream = new Windows.Storage.Streams.InMemoryRandomAccessStream())
{
await this.mediaCapture.CapturePhotoToStreamAsync(ImageEncodingProperties.CreateJpeg(), captureStream);
await captureStream.FlushAsync();
#if DEBUG
IStorageFile photoFile = await KnownFolders.PicturesLibrary.CreateFileAsync(ImageFilenameLocal, CreationCollisionOption.ReplaceExisting);
ImageEncodingProperties imageProperties = ImageEncodingProperties.CreateJpeg();
await this.mediaCapture.CapturePhotoToStorageFileAsync(imageProperties, photoFile);
#endif
string azureFilenameLatest = string.Format(this.azureStorageimageFilenameLatestFormat, currentTime);
string azureFilenameHistory = string.Format(this.azureStorageImageFilenameHistoryFormat, currentTime);
LoggingFields imageInformation = new LoggingFields();
imageInformation.AddDateTime("TakenAtUTC", currentTime);
imageInformation.AddBoolean("IsCommand", isCommand);
#if DEBUG
imageInformation.AddString("LocalFilename", photoFile.Path);
#endif
imageInformation.AddString("AzureFilenameLatest", azureFilenameLatest);
imageInformation.AddString("AzureFilenameHistory", azureFilenameHistory);
this.logging.LogEvent("Saving image(s) to Azure storage", imageInformation);
// Update the latest image in storage
if (!string.IsNullOrWhiteSpace(azureFilenameLatest))
{
captureStream.Seek(0);
Debug.WriteLine("AzureIoT Hub latest image upload start");
await this.azureIoTHubClient.UploadToBlobAsync(azureFilenameLatest, captureStream.AsStreamForRead());
Debug.WriteLine("AzureIoT Hub latest image upload done");
}
// Upload the historic image to storage
if (!string.IsNullOrWhiteSpace(azureFilenameHistory))
{
captureStream.Seek(0);
Debug.WriteLine("AzureIoT Hub historic image upload start");
await this.azureIoTHubClient.UploadToBlobAsync(azureFilenameHistory, captureStream.AsStreamForRead());
Debug.WriteLine("AzureIoT Hub historic image upload done");
}
}
}
catch (Exception ex)
{
this.logging.LogMessage("Image capture or AzureIoTHub storage upload failed " + ex.Message, LoggingLevel.Error);
}
finally
{
this.cameraBusy = false;
}
}
