public async static Task <ImageAnalysisResult> GetImageAnalysisAsync(Guid id, byte[] bytes)
{
HttpClient client = new HttpClient();
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", CognitiveConstants.ComputerVisionApiSubscriptionKey);
HttpContent payload = new ByteArrayContent(bytes);
payload.Headers.ContentType = new MediaTypeWithQualityHeaderValue("application/octet-stream");
string analysisFeatures = "Color,ImageType,Tags,Categories,Description,Adult";
var results = await client.PostAsync($"https://api.projectoxford.ai/vision/v1.0/analyze?visualFeatures={analysisFeatures}", payload);
ImageAnalysisResult result = null;
try
{
var imageAnalysisResult = JsonConvert.DeserializeObject <ImageAnalysisInfo>(await results.Content.ReadAsStringAsync());
result = new ImageAnalysisResult()
{
id = id.ToString(),
details = imageAnalysisResult,
caption = imageAnalysisResult.description.captions.FirstOrDefault().text,
tags = imageAnalysisResult.description.tags.ToList(),
};
}
catch (Exception ex)
{
}
return(result);
}
internal static void ApplyAnalysis(this ImageInformation image, ImageAnalysisResult analysis)
{
image.Caption = analysis.caption;
image.Tags = analysis.tags;
image.Categories = analysis.details.categories?.Select(s => s.name.Replace("_", " ")).ToList() ??
new List <string>();
image.IsAdult = analysis.details.adult.isAdultContent;
image.AdultScore = analysis.details.adult.adultScore;
image.IsRacy = analysis.details.adult.isRacyContent;
image.RacyScore = analysis.details.adult.racyScore;
image.Faces = (from face in analysis.details.faces
select new FaceInformation
{
Age = face.age,
Gender = face.gender
}).ToList();
image.AccentColor = $"#{analysis.details.color.accentColor}";
image.BackgroundColor = $"{analysis.details.color.dominantColorBackground}";
image.ForegroundColor = $"{analysis.details.color.dominantColorForeground}";
image.IsBlackAndWhite = analysis.details.color.isBWImg;
image.IsClipArt = analysis.details.imageType.clipArtType > 0;
image.IsLineDrawing = analysis.details.imageType.lineDrawingType > 0;
image.IsProcessing = false;
image.IsTagged = true;
}
public async static Task <ImageAnalysisResult> AnalyzeImageAsync(byte[] bytes)
{
HttpClient client = new HttpClient();
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", Common.CoreConstants.ComputerVisionApiSubscriptionKey);
var payload = new HttpBufferContent(bytes.AsBuffer());
payload.Headers.ContentType = new HttpMediaTypeHeaderValue("application/octet-stream");
string visualFeatures = "Color,ImageType,Tags,Categories,Description,Adult";
var results = await client.PostAsync(new Uri($"{Common.CoreConstants.CognitiveServicesBaseUrl}/vision/v1.0/analyze?visualFeatures={visualFeatures}"), payload);
ImageAnalysisResult result = null;
try
{
var analysisResults = await results.Content.ReadAsStringAsync();
var imageAnalysisResult = JsonConvert.DeserializeObject <ImageAnalysisInfo>(analysisResults);
result = new ImageAnalysisResult()
{
id = imageAnalysisResult.requestId,
details = imageAnalysisResult,
caption = imageAnalysisResult.description.captions.FirstOrDefault().text,
tags = imageAnalysisResult.description.tags.ToList(),
};
}
catch (Exception ex) { }
return(result);
}
internal static void LogImageAnalysisResultCode(ImageAnalysisResult resultCode, ImageAnalysisLogData logData)
{
AttachmentProcessingLogger.Initialize();
switch (resultCode)
{
case ImageAnalysisResult.ThumbnailSuccess:
AttachmentProcessingLogger.LogEvent("StorageAttachmentImageAnalysis", "ThumbnailResult", "Success", "ThumbnailOperationTime", logData.operationTimeMs.ToString());
AttachmentProcessingLogger.LogEvent("StorageAttachmentImageAnalysis", "ThumbnailSize", logData.thumbnailSize.ToString());
AttachmentProcessingLogger.LogEvent("StorageAttachmentImageAnalysis", "ThumbnailWidth", logData.thumbnailWidth.ToString(), "ThumbnailHeight", logData.thumbnailHeight.ToString());
return;
case ImageAnalysisResult.SalientRegionSuccess:
AttachmentProcessingLogger.LogEvent("StorageAttachmentImageAnalysis", "SalientRegionResult", "Success", "SalientRegionOperationTime", logData.operationTimeMs.ToString());
return;
case ImageAnalysisResult.UnknownFailure:
AttachmentProcessingLogger.LogEvent("StorageAttachmentImageAnalysis", "Failure", logData.operationTimeMs.ToString());
return;
case ImageAnalysisResult.ImageTooSmallForAnalysis:
AttachmentProcessingLogger.LogEvent("StorageAttachmentImageAnalysis", "ThumbnailResult", "ImageTooSmall", "ThumbnailOperationTime", logData.operationTimeMs.ToString());
return;
case ImageAnalysisResult.UnableToPerformSalientRegionAnalysis:
AttachmentProcessingLogger.LogEvent("StorageAttachmentImageAnalysis", "SalientRegionResult", "Failure", "SalientRegionOperationTime", logData.operationTimeMs.ToString());
return;
case ImageAnalysisResult.ImageTooBigForAnalysis:
AttachmentProcessingLogger.LogEvent("StorageAttachmentImageAnalysis", "ThumbnailResult", "ImageTooBigForAnalysis", "ImageSize", logData.thumbnailSize.ToString());
return;
default:
return;
}
}
private async Task <ImageAnalysisResult> GetImageAnalysisAsync(byte[] bytes)
{
ImageAnalysisResult result = null;
try
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", CoreConstants.ComputerVisionApiSubscriptionKey);
byte[] lowQualityImageBytes = null;
using (var data = NSData.FromArray(bytes))
{
var image = UIImage.LoadFromData(data);
var lowerQualityData = image.AsJPEG(0.1f);
lowQualityImageBytes = new byte[lowerQualityData.Length];
System.Runtime.InteropServices.Marshal.Copy(lowerQualityData.Bytes, lowQualityImageBytes, 0, Convert.ToInt32(lowerQualityData.Length));
}
var payload = new ByteArrayContent(lowQualityImageBytes);
payload.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
var analysisFeatures = "Color,ImageType,Tags,Categories,Description,Adult,Faces";
var uri = new Uri($"{CoreConstants.CognitiveServicesBaseUrl}/analyze?visualFeatures={analysisFeatures}");
using (var results = await client.PostAsync(uri, payload))
{
var analysisResults = await results.Content.ReadAsStringAsync();
var imageAnalysisResult = JsonConvert.DeserializeObject <ImageAnalysisInfo>(analysisResults);
result = new ImageAnalysisResult
{
id = Guid.NewGuid().ToString(),
details = imageAnalysisResult,
caption = imageAnalysisResult.description?.captions.FirstOrDefault()?.text,
tags = imageAnalysisResult.description?.tags.ToList()
};
if (string.IsNullOrEmpty(result.caption))
{
result.caption = "No caption";
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($"ComputerVisionService.GetImageAnalysisAsync Exception: {ex}");
}
return(result);
}
private async Task <ImageAnalysisResult> GetImageAnalysisAsync(byte[] bytes)
{
ImageAnalysisResult result = null;
try
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", CoreConstants.ComputerVisionApiSubscriptionKey);
var payload = new HttpBufferContent(bytes.AsBuffer());
payload.Headers.ContentType = new HttpMediaTypeHeaderValue("application/octet-stream");
var analysisFeatures = "Color,ImageType,Tags,Categories,Description,Adult,Faces";
var uri = new Uri($"{CoreConstants.CognitiveServicesBaseUrl}/analyze?visualFeatures={analysisFeatures}");
using (var results = await client.PostAsync(uri, payload))
{
var analysisResults = await results.Content.ReadAsStringAsync();
var imageAnalysisResult = JsonConvert.DeserializeObject <ImageAnalysisInfo>(analysisResults);
result = new ImageAnalysisResult
{
id = Guid.NewGuid().ToString(),
details = imageAnalysisResult,
caption = imageAnalysisResult.description.captions.FirstOrDefault()?.text,
tags = imageAnalysisResult.description.tags.ToList()
};
if (string.IsNullOrEmpty(result.caption))
{
result.caption = "No caption";
}
}
}
}
catch (Exception ex)
{
Debug.WriteLine($"ComputerVisionService.GetImageAnalysisAsync Exception: {ex}");
}
return(result);
}
public KeyValuePair <List <RegionRect>, ImageAnalysisResult> GetSalientRectsAsList()
{
ImageAnalysisResult imageAnalysisResult = this.EnsureSalientObjectAnalysisWrapper();
if (this.salientRegions == null && imageAnalysisResult == ImageAnalysisResult.SalientRegionSuccess)
{
this.salientRegions = new List <RegionRect>();
foreach (SalientObject salientObject in this.salientAnalysis.SalientObjects)
{
RegionRect item = new RegionRect((int)salientObject.Region.BestOutline.Left, (int)salientObject.Region.BestOutline.Top, (int)salientObject.Region.BestOutline.Right, (int)salientObject.Region.BestOutline.Bottom);
this.salientRegions.Add(item);
}
imageAnalysisResult = ImageAnalysisResult.SalientRegionSuccess;
}
else
{
imageAnalysisResult = ImageAnalysisResult.UnableToPerformSalientRegionAnalysis;
}
return(new KeyValuePair <List <RegionRect>, ImageAnalysisResult>(this.salientRegions, imageAnalysisResult));
}
public KeyValuePair <byte[], ImageAnalysisResult> GenerateThumbnail(Stream imageStream, int minImageWidth, int minImageHeight, int maxThumbnailWidth, int maxThumbnailHeight, out int width, out int height)
{
ImageAnalysisResult value = ImageAnalysisResult.UnknownFailure;
byte[] key = null;
width = 0;
height = 0;
using (Image image = Image.FromStream(imageStream, true, false))
{
if (image.Width < minImageWidth || image.Height < minImageHeight)
{
value = ImageAnalysisResult.ImageTooSmallForAnalysis;
}
else
{
this.RotateImageIfNeeded(image);
key = this.PreviewImage(image, maxThumbnailWidth, maxThumbnailHeight, out width, out height);
value = ImageAnalysisResult.ThumbnailSuccess;
}
}
return(new KeyValuePair <byte[], ImageAnalysisResult>(key, value));
}
public KeyValuePair <byte[], ImageAnalysisResult> GetSalientRectsAsByteArray()
{
ImageAnalysisResult imageAnalysisResult = this.EnsureSalientObjectAnalysisWrapper();
if (this.salientRegionsAsByte == null && imageAnalysisResult == ImageAnalysisResult.SalientRegionSuccess)
{
this.salientRegionsAsByte = new byte[this.salientAnalysis.SalientObjects.Count * 4];
int num = 0;
foreach (SalientObject salientObject in this.salientAnalysis.SalientObjects)
{
this.salientRegionsAsByte[num] = (byte)((int)salientObject.Region.BestOutline.Top * 255 / this.imageHeight);
this.salientRegionsAsByte[num + 1] = (byte)((int)salientObject.Region.BestOutline.Left * 255 / this.imageWidth);
this.salientRegionsAsByte[num + 2] = (byte)((int)salientObject.Region.BestOutline.Bottom * 255 / this.imageHeight);
this.salientRegionsAsByte[num + 3] = (byte)((int)salientObject.Region.BestOutline.Right * 255 / this.imageWidth);
num += 4;
}
imageAnalysisResult = ImageAnalysisResult.SalientRegionSuccess;
}
else
{
imageAnalysisResult = ImageAnalysisResult.UnableToPerformSalientRegionAnalysis;
}
return(new KeyValuePair <byte[], ImageAnalysisResult>(this.salientRegionsAsByte, imageAnalysisResult));
}
public async Task <ImageAnalysisResult> AnalyzeImageAsync(string url)
{
var analysisResult = new ImageAnalysisResult();
try
{
// USING Microsoft provided VisionClientLibrary seems not working in NET Core as-is, a fix is required for ExpandoObject
// see: https://github.com/Microsoft/Cognitive-Vision-DotNetCore/pull/1/commits/9c4647edb400aecd4def330537d5bcd74f126111
Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeImageAsync(): initializing VisionAPI client");
var visionApiClient = new VisionServiceClient(m_VisionAPISubscriptionKey, "https://westeurope.api.cognitive.microsoft.com/vision/v1.0");
var visualFeatures = new List <VisualFeature> {
VisualFeature.Adult, VisualFeature.Categories, VisualFeature.Color, VisualFeature.Description, VisualFeature.Faces, VisualFeature.ImageType /*, VisualFeature.Tags */
};
var details = new List <string> {
"Celebrities", "Landmarks"
};
Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeImageAsync(): started image analysis");
var visionApiResult = await visionApiClient.AnalyzeImageAsync(url, visualFeatures, details).ConfigureAwait(false);
Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeImageAsync(): executing OCR");
var ocrResult = await visionApiClient.RecognizeTextAsync(url).ConfigureAwait(false);
Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeImageAsync(): performing tag identification");
var tagsResult = await visionApiClient.GetTagsAsync(url).ConfigureAwait(false);
Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeImageAsync(): analysis completed");
// Mapping VisionAPI Client entity to domain entity
analysisResult.AdultContent = new ImageAnalysisAdultContentResult {
AdultScore = visionApiResult.Adult.AdultScore, IsAdultContent = visionApiResult.Adult.IsAdultContent, IsRacyContent = visionApiResult.Adult.IsRacyContent, RacyScore = visionApiResult.Adult.RacyScore
};
analysisResult.Colors = new ImageAnalysisColorResult {
AccentColor = visionApiResult.Color.AccentColor, DominantColorBackground = visionApiResult.Color.DominantColorBackground, DominantColorForeground = visionApiResult.Color.DominantColorForeground, IsBWImg = visionApiResult.Color.IsBWImg
};
analysisResult.Categories = visionApiResult.Categories.Select(c => new ImageAnalysisCategoryResult {
Text = c.Name, Score = c.Score
}).OrderByDescending(c => c.Score).ToList();
analysisResult.Descriptions = visionApiResult.Description.Captions.Select(c => new ImageAnalysisDescriptionResult {
Text = c.Text, Score = c.Confidence
}).OrderByDescending(c => c.Score).ToList();
// Merge detected tags from image analysis and image tags
analysisResult.Tags = tagsResult.Tags.Select(t => new ImageAnalysisTagResult {
Text = t.Name, Score = t.Confidence, Hint = t.Hint
}).ToList();
foreach (var t in visionApiResult.Description.Tags)
{
analysisResult.Tags.Add(new ImageAnalysisTagResult {
Text = t, Score = 0.0, Hint = string.Empty
});
}
analysisResult.Faces = visionApiResult.Faces.Select(f => new ImageAnalysisFaceResult {
Age = f.Age, Gender = f.Gender == "Male" ? Gender.Male : f.Gender == "female" ? Gender.Female : Gender.Unknown
}).ToList();
analysisResult.Text = ocrResult.Regions.Select(r => new ImageAnalysisTextResult()
{
Language = ocrResult.Language, Orientation = ocrResult.Orientation, TextAngle = ocrResult.TextAngle.GetValueOrDefault(), Text = string.Join(" ", r.Lines.Select(l => string.Join(" ", l.Words.Select(w => w.Text))))
}).ToList();
// Extend analysis by estimating reading time for each transcribed text
foreach (var text in analysisResult.Text)
{
text.WordCount = TextTokenizer.GetWordCount(text.Text);
text.ReadingTimeInMinutes = ReadingTimeEstimator.GetEstimatedReadingTime(text.WordCount, text.Language);
analysisResult.WatchingTimeInMinutes += text.ReadingTimeInMinutes;
}
// Add an additional default time for estimating how long it will take to the user to watch the picture
analysisResult.WatchingTimeInMinutes += DefaultImageWatchingTime;
}
catch (Exception ex)
{
Console.WriteLine($"\t\t\tContentAnalyzer.AnalyzeImageAsync(): an error occured while analyzing image - {ex.Message}");
}
return(analysisResult);
}
static async Task <ImageAnalysisResult> AnalyzeImage(string filePath, string factoryId, string cameraId, EnvSettings.AIModule module, EnvSettings.AIModule.Tag tag, string outputFolder)
{
try
{
ImageAnalysisResult analyzeResult = null;
// Get output directory details
string storageAccountName = _envSettings.GetProperty("StorageAccountName");
string dbeShareContainerName = _envSettings.GetProperty("DBEShareContainerName");
string flaggedFolder = "flagged";
string nonFlaggedFolder = "safe";
// Read image
RecognitionResults recognitionResults = null;
byte[] byteArray = File.ReadAllBytes(filePath);
using (ByteArrayContent content = new ByteArrayContent(byteArray))
{
content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
try
{
var response = await _httpClient.PostAsync(module.ScoringEndpoint, content);
if (!response.IsSuccessStatusCode)
{
_consoleLogger.LogError($"Failed to make POST request to module {module.ScoringEndpoint}. Response: {response.ReasonPhrase}");
return(null);
}
else
{
_consoleLogger.LogDebug($"POST request to module {module.ScoringEndpoint} was successful");
var contentString = await response.Content.ReadAsStringAsync();
recognitionResults = JsonConvert.DeserializeObject <RecognitionResults>(contentString);
}
}
catch (Exception e)
{
_consoleLogger.LogError($"AnalyzeImage:PostAsync failed to make POST request to module {module.ScoringEndpoint}. Exception: {e}");
return(null);
}
}
/// Need to differentiate between the current tag being flagged and
/// any other tags from this module in order to mark the image appropriately.
/// Logic invites to think that in case the current tag is flagged,
/// it will also be in the all flagged tags list.
var currentFlaggedTag = recognitionResults.Predictions.Where(x => x.TagName == tag.Name && x.Probability >= tag.Probability);
var allFlaggedTags = recognitionResults.Predictions.Where(x => module.Tags.Where(y => x.TagName == y.Name && x.Probability >= y.Probability).Count() > 0);
// Create analyze result object
string fileName = Path.GetFileName(filePath);
if (currentFlaggedTag.Count() > 0)
{
string imageUri = $"https://{storageAccountName}.blob.core.windows.net/{dbeShareContainerName}/{factoryId}/{cameraId}/{flaggedFolder}/{fileName}";
_consoleLogger.LogInformation($"---> Found tags in image {filePath}: {string.Join(", ", currentFlaggedTag.Select(x => x.TagName))}");
// Create message content
string datePattern = @"^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{3})$";
var match = Regex.Match(Path.GetFileNameWithoutExtension(fileName), datePattern);
DateTime timestamp = new DateTime(
Convert.ToInt32(match.Groups[1].Value),
Convert.ToInt32(match.Groups[2].Value),
Convert.ToInt32(match.Groups[3].Value),
Convert.ToInt32(match.Groups[4].Value),
Convert.ToInt32(match.Groups[5].Value),
Convert.ToInt32(match.Groups[6].Value),
Convert.ToInt32(match.Groups[7].Value));
analyzeResult = new ImageAnalysisResult()
{
ImageUri = imageUri,
Timestamp = timestamp,
Results = ImageAnalysisResult.Result.Results(currentFlaggedTag, module),
};
// Get flat results for reporting purposes
FlatImageAnalysisResult[] flatImageResults = FlatImageAnalysisResult.Convert(factoryId, cameraId, analyzeResult);
foreach (var flatResult in flatImageResults)
{
// Create hub message and set its properties
var message = new Message(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(flatResult)));
message.Properties.Add("messageType", "reporting");
// Send reporting message
await SendMessageToHub(message);
// Log it
_consoleLogger.LogTrace($"Sent reporting message for camera {cameraId}");
}
}
else
{
_consoleLogger.LogTrace($"No tags were found in image {filePath}");
}
// Save image to output directory
string destinationFolder = allFlaggedTags.Count() > 0 ? flaggedFolder : nonFlaggedFolder;
// Set output directory
string outputDirectory = Path.Combine(outputFolder, factoryId, cameraId, destinationFolder);
if (!Directory.Exists(outputDirectory))
{
Directory.CreateDirectory(outputDirectory);
}
// Save image
string imageOutputPath = Path.Combine(outputDirectory, fileName);
File.WriteAllBytes(imageOutputPath, byteArray);
_consoleLogger.LogTrace($"Moving image to final destination folder {imageOutputPath}");
// Save payload
string fileOutputPath = Path.Combine(outputDirectory, Path.ChangeExtension(fileName, "json"));
File.WriteAllText(fileOutputPath, JsonConvert.SerializeObject(recognitionResults.Predictions));
// Delete image from local folder
File.Delete(filePath);
return(analyzeResult);
}
catch (Exception e)
{
_consoleLogger.LogCritical("AnalyzeImage caught an exception: {0}", e);
return(null);
}
}
