/// <inheritdoc/>
public async Task ExecuteAsync(CancellationToken cancellationToken = default)
{
if (!_conceptValues.Any())
{
// Do not call translator service if there is nothing to translate
return;
}
_translations = await _service.TranslateAsync(_conceptValues, _clientName, cancellationToken);
}
public async Task <CognitiveResult> RecognizeAsync(Stream stream, string language, RecognitionType recognitionType = RecognitionType.Vision | RecognitionType.Emotion, Func <RecognitionPhase, Task> onProgress = null)
{
await this.RaiseOnProgressAsync(onProgress, RecognitionPhase.QueryingService);
var visionService = new VisionServiceClient(Settings.VisionSubscriptionKey);
var result = new CognitiveResult();
if (recognitionType.HasFlag(RecognitionType.Vision) || recognitionType.HasFlag(RecognitionType.Emotion))
{
var imageBytes = await stream.ToArrayAsync();
var features = new HashSet <VisualFeature> {
VisualFeature.Description
};
if (recognitionType.HasFlag(RecognitionType.Emotion))
{
features.Add(VisualFeature.Faces);
}
var visionSettings = await VisionSettingsProvider?.GetSettingsAsync();
var analyzeImageResult = await visionService.AnalyzeImageAsync(stream, features);
var visionResult = result.VisionResult;
Caption originalDescription;
Caption filteredDescription;
var isValid = analyzeImageResult.IsValid(out originalDescription, out filteredDescription, visionSettings);
visionResult.IsValid = isValid;
visionResult.RawDescription = originalDescription.Text;
visionResult.Confidence = originalDescription.Confidence;
if (isValid)
{
visionResult.Description = filteredDescription.Text;
if (language != DefaultLanguge && IsTranslatorServiceRegistered)
{
// Make sure to use the updated translator subscription key.
translatorService.SubscriptionKey = Settings.TranslatorSubscriptionKey;
// The description needs to be translated.
await this.RaiseOnProgressAsync(onProgress, RecognitionPhase.Translating);
var translation = await translatorService.TranslateAsync(filteredDescription.Text, from : DefaultLanguge, to : language);
visionResult.TranslatedDescription = translation;
}
}
if (recognitionType.HasFlag(RecognitionType.Emotion))
{
// If there is one or more faces, asks the service information about them.
if (IsEmotionServiceRegistered && (analyzeImageResult.Faces?.Any() ?? false))
{
await this.RaiseOnProgressAsync(onProgress, RecognitionPhase.RecognizingFaces);
var emotionService = new EmotionServiceClient(Settings.EmotionSubscriptionKey);
foreach (var face in analyzeImageResult.Faces)
{
using (var ms = new MemoryStream(imageBytes))
{
var emotions = await emotionService.RecognizeAsync(ms, face.FaceRectangle.ToRectangle());
var emotionResult = emotions.GetEmotionResult(face);
result.EmotionResults.Add(emotionResult);
}
}
}
}
}
if (recognitionType.HasFlag(RecognitionType.Text))
{
await this.RaiseOnProgressAsync(onProgress, RecognitionPhase.RecognizingText);
var results = await visionService.RecognizeTextAsync(stream);
var text = results.GetRecognizedText();
result.OcrResult.Text = text;
}
return(result);
}
public static async Task <TranslationOrchestrationContext> TranslateDocument([ActivityTrigger] TranslationOrchestrationContext context, [Inject] ITranslatorService service, ILogger log)
{
context.TranslatedText = await service.TranslateAsync(context.SourceText, context.Language);
return(context);
}
public async Task DescribeImageAsync()
{
IsBusy = true;
string message = null;
if (IsOnline)
{
try
{
MessengerInstance.Send(new NotificationMessage(Constants.PhotoTaken));
using (var stream = await streamingService.GetCurrentFrameAsync())
{
if (stream != null)
{
StatusMessage = AppResources.QueryingVisionService;
var result = await visionServiceClient.DescribeAsync(stream);
StatusMessage = AppResources.VisionServiceQueried;
if (result.Description.Captions.Length > 0)
{
message = result.Description.Captions.First().Text;
StatusMessage = message;
if (Settings.AutomaticTranslation && Language != Constants.DefaultLanguge)
{
// The description needs to be translated.
StatusMessage = AppResources.Translating;
message = await translatorService.TranslateAsync(message);
}
}
else
{
message = AppResources.RecognitionFailed;
}
}
else
{
message = AppResources.UnableToGetImage;
}
}
}
catch (Exception ex)
{
var msg = ex.Message;
Debug.WriteLine(msg);
message = AppResources.RecognitionError;
}
}
else
{
// Internet isn't available, to service cannot be reached.
message = AppResources.NoConnection;
}
// Speaks the result.
StatusMessage = message;
await speechService.SpeechAsync(message, Language);
IsBusy = false;
}
public async Task DescribeImageAsync()
{
IsBusy = true;
StatusMessage = null;
string baseDescription = null;
string facesRecognizedDescription = null;
string emotionDescription = null;
MessengerInstance.Send(new NotificationMessage(Constants.TakingPhoto));
try
{
StatusMessage = AppResources.QueryingVisionService;
using (var stream = await streamingService.GetCurrentFrameAsync())
{
if (stream != null)
{
if (await Network.IsInternetAvailableAsync())
{
var imageBytes = await stream.ToArrayAsync();
MessengerInstance.Send(new NotificationMessage <byte[]>(imageBytes, Constants.PhotoTaken));
var visualFeatures = new VisualFeature[] { VisualFeature.Description, VisualFeature.Faces };
var result = await visionService.AnalyzeImageAsync(stream, visualFeatures);
Caption originalDescription;
Caption filteredDescription;
if (result.IsValid(out originalDescription, out filteredDescription))
{
baseDescription = filteredDescription.Text;
if (Language != Constants.DefaultLanguge && IsTranslatorServiceRegistered)
{
// The description needs to be translated.
StatusMessage = AppResources.Translating;
var translation = await translatorService.TranslateAsync(filteredDescription.Text, from : Constants.DefaultLanguge, to : Language);
if (Settings.ShowOriginalDescriptionOnTranslation)
{
baseDescription = $"{translation} ({filteredDescription.Text})";
}
else
{
baseDescription = translation;
}
}
if (Settings.ShowDescriptionConfidence)
{
baseDescription = $"{baseDescription} ({Math.Round(filteredDescription.Confidence, 2)})";
}
try
{
// If there is one or more faces, asks the service information about them.
if (IsEmotionServiceRegistered && result.Faces?.Count() > 0)
{
StatusMessage = AppResources.RecognizingFaces;
var messages = new StringBuilder();
foreach (var face in result.Faces)
{
using (var ms = new MemoryStream(imageBytes))
{
var emotions = await emotionService.RecognizeAsync(ms, face.FaceRectangle.ToRectangle());
var bestEmotion = emotions.FirstOrDefault()?.Scores.GetBestEmotion();
// Creates the emotion description text to be speeched (if there are interesting information).
var emotionMessage = SpeechHelper.GetEmotionMessage(face, bestEmotion, includeAge: Settings.GuessAge);
if (!string.IsNullOrWhiteSpace(emotionMessage))
{
messages.Append(emotionMessage);
}
}
}
// Checks if at least one emotion has been actually recognized.
if (messages.Length > 0)
{
// Describes how many faces have been recognized.
if (result.Faces.Count() == 1)
{
facesRecognizedDescription = AppResources.FaceRecognizedSingular;
}
else
{
facesRecognizedDescription = $"{string.Format(AppResources.FacesRecognizedPlural, result.Faces.Count())} {Constants.SentenceEnd}";
}
emotionDescription = messages.ToString();
}
}
}
catch (Microsoft.ProjectOxford.Common.ClientException ex) when(ex.Error.Code.ToLower() == "unauthorized")
{
// Unable to access the service (tipically, due to invalid registration keys).
baseDescription = AppResources.UnableToAccessService;
}
catch
{ }
}
else
{
if (Settings.ShowRawDescriptionOnInvalidRecognition && originalDescription != null)
{
baseDescription = $"{AppResources.RecognitionFailed} ({originalDescription.Text}, {Math.Round(originalDescription.Confidence, 2)})";
}
else
{
baseDescription = AppResources.RecognitionFailed;
}
}
}
else
{
// Internet isn't available, the service cannot be reached.
baseDescription = AppResources.NoConnection;
}
}
else
{
baseDescription = AppResources.UnableToTakePhoto;
}
}
}
catch (WebException)
{
// Internet isn't available, the service cannot be reached.
baseDescription = AppResources.NoConnection;
}
catch (ClientException)
{
// Unable to access the service (tipically, due to invalid registration keys).
baseDescription = AppResources.UnableToAccessService;
}
catch (Exception ex)
{
var error = AppResources.RecognitionError;
if (Settings.ShowExceptionOnError)
{
error = $"{error} ({ex.Message})";
}
baseDescription = error;
}
// Shows and speaks the result.
var message = $"{baseDescription}{Constants.SentenceEnd} {facesRecognizedDescription} {emotionDescription}";
StatusMessage = this.GetNormalizedMessage(message);
await SpeechHelper.TrySpeechAsync(message);
IsBusy = false;
}
