private async Task <ImagePrediction> CustomVisionAnalyzeImageByStreamAsync(Stream imageStream, Guid projectId, string publishedName, CustomVisionProjectType projectType)
{
ImagePrediction imagePrediction = null;
switch (projectType)
{
case CustomVisionProjectType.Classification:
imagePrediction = await _customVisionPredictionClient.ClassifyImageWithNoStoreAsync(
projectId : projectId,
publishedName : publishedName,
imageData : imageStream);
break;
case CustomVisionProjectType.Object_Detection:
imagePrediction = await _customVisionPredictionClient.DetectImageWithNoStoreAsync(
projectId : projectId,
publishedName : publishedName,
imageData : imageStream);
break;
}
return(imagePrediction);
}
async Task Ok()
{
IsEnabled = false;
try
{
var client = new CustomVisionPredictionClient
{
ApiKey = PredictionKey,
Endpoint = Endpoint
};
if (Guid.TryParse(ProjectId, out var pId))
{
var imagePath = "ObjectDetector.Images.single.png";
var assembly = typeof(SettingsViewModel).GetTypeInfo().Assembly;
using (var stream = assembly.GetManifestResourceStream(imagePath))
{
await client.DetectImageWithNoStoreAsync(pId, publishName, stream);
}
await KeyService.SetPredictionKey(PredictionKey);
await KeyService.SetProjectId(ProjectId);
await KeyService.SetPublishName(PublishName);
await KeyService.SetEndpoint(Endpoint);
Analytics.TrackEvent("Updating keys");
await Application.Current.MainPage.Navigation.PopModalAsync();
}
else
{
Analytics.TrackEvent("Failed updating keys", new Dictionary <string, string> {
{ "Error", "The project Id is not a valid GUID" }
});
await Application.Current.MainPage.DisplayAlert("Error", "The project Id is not a valid GUID", "OK");
}
}
catch (Exception ex)
{
Analytics.TrackEvent("Failed updating keys", new Dictionary <string, string> {
{ "Error", "The project Id and prediction key don't match an existing project" }
});
Crashes.TrackError(ex, new Dictionary <string, string> {
{ "Action", "Testing key and project id" }
});
await Application.Current.MainPage.DisplayAlert("Error", "The project Id and prediction key don't match an existing project", "OK");
}
finally
{
IsEnabled = true;
}
}
public async Task Execute(IChatService chatService, string userName, ReadOnlyMemory <char> rhs)
{
if (string.IsNullOrEmpty(IterationName))
{
await IdentifyIterationName();
}
var client = new CustomVisionPredictionClient()
{
ApiKey = _CustomVisionKey,
Endpoint = _AzureEndpoint
};
var obsImage = await _TrainHat.GetScreenshotFromObs();
////////////////////////////
ImagePrediction result;
try
{
result = await client.DetectImageWithNoStoreAsync(_AzureProjectId, IterationName, obsImage);
} catch (CustomVisionErrorException ex) {
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
await IdentifyIterationName();
}
await chatService.SendMessageAsync("Unable to detect Fritz's hat right now... please try again in 1 minute");
return;
}
var bestMatch = result.Predictions.OrderByDescending(p => p.Probability).FirstOrDefault();
if (bestMatch == null || bestMatch.Probability <= 0.3D)
{
await chatService.SendMessageAsync("csharpAngry 404 Hat Not Found! Let's ask a moderator to !addhat so we can identify it next time");
// do we store the image?
return;
}
await chatService.SendMessageAsync($"csharpClip I think (with {bestMatch.Probability.ToString("0.0%")} certainty) Jeff is currently wearing his {bestMatch.TagName} hat csharpClip");
}
public async Task <List <PhotoFunFact> > FunFact()
{
ICustomVisionPredictionClient _customVisionPredictionClient = new CustomVisionPredictionClient
{
Endpoint = _appSettings.Value.CustomVisionPredictionEndPoint,
ApiKey = _appSettings.Value.CustomVisionPredictionKey
};
var imageAnalysis = await _customVisionPredictionClient.DetectImageWithNoStoreAsync(
new Guid(_appSettings.Value.CustomVisionPredictionProjectId),
_appSettings.Value.CustomVisionPredictionIterationName,
Request.Body
);
return(imageAnalysis.Predictions.Select(p => new PhotoFunFact()
{
TagName = p.TagName,
Probability = (int)Math.Floor(p.Probability * 100)
}).ToList());
}
public async Task Execute(IChatService chatService, string userName, ReadOnlyMemory <char> rhs)
{
if (string.IsNullOrEmpty(IterationName))
{
await IdentifyIterationName();
}
var client = new CustomVisionPredictionClient()
{
ApiKey = _CustomVisionKey,
Endpoint = _AzureEndpoint,
};
await _HubContext.Clients.All.SendAsync("shutter");
var obsImage = await _TrainHat.GetScreenshotFromObs();
////////////////////////////
ImagePrediction result;
try
{
result = await client.DetectImageWithNoStoreAsync(_AzureProjectId, IterationName, obsImage);
} catch (CustomVisionErrorException ex) {
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
await IdentifyIterationName();
}
await chatService.SendMessageAsync("Unable to detect Fritz's hat right now... please try again in 1 minute");
return;
}
if (DateTime.UtcNow.Subtract(result.Created).TotalSeconds > Cooldown.Value.TotalSeconds)
{
await chatService.SendMessageAsync($"I previously predicted this hat about {DateTime.UtcNow.Subtract(result.Created).TotalSeconds} seconds ago");
}
var bestMatch = result.Predictions.OrderByDescending(p => p.Probability).FirstOrDefault();
if (bestMatch == null || bestMatch.Probability < 0.7D)
{
await chatService.SendMessageAsync("csharpAngry 404 Hat Not Found! Let's ask a moderator to !addhat so we can identify it next time");
// do we store the image?
return;
}
var hatData = (await _Repository.GetHatData(bestMatch.TagName));
var nameToReport = (hatData == null ? bestMatch.TagName : (string.IsNullOrEmpty(hatData.Name) ? bestMatch.TagName : hatData.Name));
await chatService.SendMessageAsync($"csharpClip I think (with {bestMatch.Probability.ToString("0.0%")} certainty) Jeff is currently wearing his {nameToReport} hat csharpClip");
if (hatData != null && !string.IsNullOrEmpty(hatData.Description))
{
await chatService.SendMessageAsync(hatData.Description);
}
await _HubContext.Clients.All.SendAsync("hatDetected", bestMatch.Probability.ToString("0.0%"), bestMatch.TagName, nameToReport, hatData?.Description);
}
private async Task ProcessImage(SoftwareBitmap image)
{
try
{
Func <Task <Stream> > imageStreamCallback;
using (InMemoryRandomAccessStream stream = new InMemoryRandomAccessStream())
{
BitmapEncoder encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.JpegEncoderId, stream);
encoder.SetSoftwareBitmap(image);
await encoder.FlushAsync();
// Read the pixel bytes from the memory stream
using (var reader = new DataReader(stream.GetInputStreamAt(0)))
{
var bytes = new byte[stream.Size];
await reader.LoadAsync((uint)stream.Size);
reader.ReadBytes(bytes);
imageStreamCallback = () => Task.FromResult <Stream>(new MemoryStream(bytes));
}
}
Microsoft.Azure.CognitiveServices.Vision.ComputerVision.ComputerVisionClient visionClient = new Microsoft.Azure.CognitiveServices.Vision.ComputerVision.ComputerVisionClient(
new ApiKeyServiceClientCredentials(settings.ComputerVisionKey),
new System.Net.Http.DelegatingHandler[] { });
// Create a prediction endpoint, passing in the obtained prediction key
CustomVisionPredictionClient customVisionClient = new CustomVisionPredictionClient()
{
ApiKey = settings.CustomVisionKey,
Endpoint = $"https://{settings.CustomVisionRegion}.api.cognitive.microsoft.com"
};
Microsoft.Azure.CognitiveServices.Vision.Face.FaceClient faceClient = new Microsoft.Azure.CognitiveServices.Vision.Face.FaceClient(
new ApiKeyServiceClientCredentials(settings.FaceKey),
new System.Net.Http.DelegatingHandler[] { });
visionClient.Endpoint = settings.ComputerVisionEndpoint;
faceClient.Endpoint = settings.FaceEndpoint;
List <VisualFeatureTypes> features =
new List <VisualFeatureTypes>()
{
VisualFeatureTypes.Categories, VisualFeatureTypes.Description,
VisualFeatureTypes.Tags, VisualFeatureTypes.Faces, VisualFeatureTypes.Brands
};
// The list of Face attributes to return.
IList <FaceAttributeType> faceAttributes =
new FaceAttributeType[]
{
FaceAttributeType.Gender, FaceAttributeType.Age,
FaceAttributeType.Smile, FaceAttributeType.Emotion,
FaceAttributeType.Glasses, FaceAttributeType.Hair
};
try
{
if (!imageAnalysisRunning && DateTime.Now.Subtract(imageAnalysisLastDate).TotalMilliseconds > 1000)
{
imageAnalysisRunning = true;
_ = Task.Run(async() =>
{
ImageAnalysis analysis = await visionClient.AnalyzeImageInStreamAsync(await imageStreamCallback(), features);
ImagePrediction analysisCV = null;
try
{
analysisCV = await customVisionClient.DetectImageWithNoStoreAsync(new Guid(settings.CustomVisionProjectId), settings.CustomVisionIterationName, await imageStreamCallback());
}
catch (Exception)
{
// Throw away error
}
UpdateWithAnalysis(analysis, analysisCV);
imageAnalysisLastDate = DateTime.Now;
imageAnalysisRunning = false;
});
}
var analysisFace = await faceClient.Face.DetectWithStreamWithHttpMessagesAsync(await imageStreamCallback(), returnFaceId : true, returnFaceAttributes : faceAttributes);
imageWidth = image.PixelWidth;
imageHeight = image.PixelHeight;
facesControl.UpdateEvent(new CognitiveEvent()
{
Faces = analysisFace.Body, ImageWidth = image.PixelWidth, ImageHeight = image.PixelHeight
});
if (analysisFace.Body.Count() > 0 && settings.DoFaceDetection)
{
var groups = await faceClient.PersonGroup.ListWithHttpMessagesAsync();
var group = groups.Body.FirstOrDefault(x => x.Name == settings.GroupName);
if (group != null)
{
var results = await faceClient.Face.IdentifyWithHttpMessagesAsync(analysisFace.Body.Select(x => x.FaceId.Value).ToArray(), group.PersonGroupId);
foreach (var identifyResult in results.Body)
{
var cand = identifyResult.Candidates.FirstOrDefault(x => x.Confidence > settings.FaceThreshold / 100d);
if (cand == null)
{
Console.WriteLine("No one identified");
}
else
{
// Get top 1 among all candidates returned
var candidateId = cand.PersonId;
var person = await faceClient.PersonGroupPerson.GetWithHttpMessagesAsync(group.PersonGroupId, candidateId);
tagsControl.UpdateEvent(new CognitiveEvent()
{
IdentifiedPerson = person.Body, IdentifiedPersonPrediction = cand.Confidence
});
Console.WriteLine("Identified as {0}", person.Body.Name);
}
}
}
}
}
catch (Exception)
{
// Eat exception
}
}
catch (Exception)
{
// eat this exception too
}
}