/// <summary>
/// Submits text analytics requests depending on the settings (sentimentAnalysisSetting). Every utterance will be submitted independently to text analytics.
/// (This means for instance that every utterance will have a separate sentiment score).
/// </summary>
/// <param name="speechTranscript">The speech transcript object.</param>
/// <param name="sentimentAnalysisSetting">The sentiment analysis setting.</param>
/// <returns>The job ids and errors, if any were found.</returns>
public async Task <(IEnumerable <string> jobIds, IEnumerable <string> errors)> SubmitUtteranceLevelRequests(
SpeechTranscript speechTranscript,
SentimentAnalysisSetting sentimentAnalysisSetting)
{
speechTranscript = speechTranscript ?? throw new ArgumentNullException(nameof(speechTranscript));
if (sentimentAnalysisSetting != SentimentAnalysisSetting.UtteranceLevel)
{
return(new List <string>(), new List <string>());
}
var documents = speechTranscript.RecognizedPhrases.Where(r => r.NBest.FirstOrDefault() != null && !string.IsNullOrEmpty(r.NBest.First().Display)).Select(r => new TextDocumentInput($"{r.Channel}_{r.Offset}", r.NBest.First().Display)
{
Language = Locale
});
var actions = new TextAnalyticsActions
{
DisplayName = "IngestionClient",
AnalyzeSentimentActions = new List <AnalyzeSentimentAction>()
{
new AnalyzeSentimentAction()
}
};
return(await SubmitDocumentsAsync(documents, actions).ConfigureAwait(false));
}
/// <summary>
/// Submits text analytics requests depending on the settings (sentimentAnalysisSetting). The whole transcript (per channel) will be submitted in a single request.
/// (This means for instance that one single sentiment score will be generated per channel).
/// </summary>
/// <param name="speechTranscript">The speech transcript object.</param>
/// <param name="sentimentAnalysisSetting">The sentiment analysis setting.</param>
/// <param name="piiRedactionSetting">The PII redaction setting.</param>
/// <returns>The job ids and errors, if any were found.</returns>
public async Task <(IEnumerable <string> jobIds, IEnumerable <string> errors)> SubmitAudioLevelRequests(
SpeechTranscript speechTranscript,
SentimentAnalysisSetting sentimentAnalysisSetting,
PiiRedactionSetting piiRedactionSetting)
{
speechTranscript = speechTranscript ?? throw new ArgumentNullException(nameof(speechTranscript));
if (sentimentAnalysisSetting != SentimentAnalysisSetting.AudioLevel && piiRedactionSetting != PiiRedactionSetting.UtteranceAndAudioLevel)
{
return(new List <string>(), new List <string>());
}
var documents = speechTranscript.CombinedRecognizedPhrases.Where(r => !string.IsNullOrEmpty(r.Display)).Select(r => new TextDocumentInput($"{r.Channel}", r.Display)
{
Language = Locale
});
var actions = new TextAnalyticsActions
{
DisplayName = "IngestionClient"
};
if (sentimentAnalysisSetting == SentimentAnalysisSetting.AudioLevel)
{
actions.AnalyzeSentimentActions = new List <AnalyzeSentimentAction>()
{
new AnalyzeSentimentAction()
};
}
if (piiRedactionSetting == PiiRedactionSetting.UtteranceAndAudioLevel)
{
var action = new RecognizePiiEntitiesAction();
if (!string.IsNullOrEmpty(FetchTranscriptionEnvironmentVariables.PiiCategories))
{
var piiEntityCategories = FetchTranscriptionEnvironmentVariables.PiiCategories.Split(",").Select(c => new PiiEntityCategory(c));
foreach (var category in piiEntityCategories)
{
action.CategoriesFilter.Add(category);
}
}
actions.RecognizePiiEntitiesActions = new List <RecognizePiiEntitiesAction>()
{
action
};
}
return(await SubmitDocumentsAsync(documents, actions).ConfigureAwait(false));
}
public async Task <IEnumerable <string> > AddUtteranceLevelEntitiesAsync(
SpeechTranscript speechTranscript,
SentimentAnalysisSetting sentimentAnalysisSetting)
{
speechTranscript = speechTranscript ?? throw new ArgumentNullException(nameof(speechTranscript));
var errors = new List <string>();
if (sentimentAnalysisSetting != SentimentAnalysisSetting.UtteranceLevel)
{
return(errors);
}
var documents = speechTranscript.RecognizedPhrases.Where(r => r.NBest.FirstOrDefault() != null && !string.IsNullOrEmpty(r.NBest.First().Display)).Select(r => new TextDocumentInput($"{r.Channel}_{r.Offset}", r.NBest.First().Display)
{
Language = Locale
});
var actions = new TextAnalyticsActions
{
DisplayName = "IngestionClient",
AnalyzeSentimentActions = new List <AnalyzeSentimentAction>()
{
new AnalyzeSentimentAction()
}
};
var(sentimentResults, piiResults, requestErrors) = await this.GetDocumentResultsAsync(documents, actions).ConfigureAwait(false);
errors.AddRange(requestErrors);
foreach (var recognizedPhrase in speechTranscript.RecognizedPhrases)
{
var index = $"{recognizedPhrase.Channel}_{recognizedPhrase.Offset}";
var firstNBest = recognizedPhrase.NBest.FirstOrDefault();
var sentimentResult = sentimentResults.Where(s => s.Id == index).FirstOrDefault();
if (firstNBest != null)
{
firstNBest.Sentiment = new Sentiment()
{
Negative = sentimentResult?.DocumentSentiment.ConfidenceScores.Negative ?? 0.0,
Positive = sentimentResult?.DocumentSentiment.ConfidenceScores.Positive ?? 0.0,
Neutral = sentimentResult?.DocumentSentiment.ConfidenceScores.Neutral ?? 0.0,
};
}
}
return(errors);
}
public static double GetCostEstimation(
TimeSpan timeSpan,
int numberOfChannels,
bool isCustomModel,
SentimentAnalysisSetting sentimentSetting,
PiiRedactionSetting entityRedactionSetting)
{
double costPerHour = isCustomModel ? STTCustomModelCostPerHour : STTCostPerHour;
var price = timeSpan.TotalHours * costPerHour;
if (sentimentSetting != SentimentAnalysisSetting.None)
{
price += timeSpan.TotalHours * TextAnalyticsCostPerHour;
}
if (entityRedactionSetting != PiiRedactionSetting.None)
{
price += timeSpan.TotalHours * TextAnalyticsCostPerHour;
}
price *= numberOfChannels;
return(price);
}
public async Task <IEnumerable <string> > AddSentimentToTranscriptAsync(SpeechTranscript speechTranscript, SentimentAnalysisSetting sentimentSetting)
{
if (speechTranscript == null)
{
throw new ArgumentNullException(nameof(speechTranscript));
}
var sentimentErrors = new List <string>();
try
{
var textAnalyticsChunks = new List <TextAnalyticsRequestsChunk>();
if (sentimentSetting == SentimentAnalysisSetting.UtteranceLevel)
{
textAnalyticsChunks = CreateUtteranceLevelRequests(speechTranscript, SentimentRequestLimit);
}
else if (sentimentSetting == SentimentAnalysisSetting.AudioLevel)
{
textAnalyticsChunks = CreateAudioLevelRequests(speechTranscript, SentimentRequestLimit);
}
var responses = new List <HttpResponseMessage>();
foreach (var chunk in textAnalyticsChunks)
{
var chunkString = JsonConvert.SerializeObject(chunk);
var response = await MakeRequestAsync(chunkString, SentimentSuffix).ConfigureAwait(false);
responses.Add(response);
}
Log.LogInformation($"Total responses: {responses.Count}");
sentimentErrors = await AddSentimentToSpeechTranscriptAsync(responses, speechTranscript, sentimentSetting).ConfigureAwait(false);
return(sentimentErrors);
}
catch (Exception e)
{
var sentimentError = $"Sentiment Analysis failed with exception: {e.Message}";
Log.LogError(sentimentError);
sentimentErrors.Add(sentimentError);
return(sentimentErrors);
}
}
private async Task <List <string> > AddSentimentToSpeechTranscriptAsync(List <HttpResponseMessage> responses, SpeechTranscript speechTranscript, SentimentAnalysisSetting sentimentSetting)
{
var sentimentErrors = new List <string>();
if (!speechTranscript.RecognizedPhrases.Any())
{
return(sentimentErrors);
}
var textAnalyticsDocuments = new List <TextAnalyticsDocument>();
foreach (var message in responses)
{
message.EnsureSuccessStatusCode();
var responseBody = await message.Content.ReadAsStringAsync().ConfigureAwait(false);
var textAnalyticsResponse = JsonConvert.DeserializeObject <TextAnalyticsResponse>(responseBody);
foreach (var error in textAnalyticsResponse.Errors)
{
var errorMessage = $"Sentiment extraction failed with error: {error.Error.InnerError.Message}";
Log.LogError(errorMessage);
sentimentErrors.Add(errorMessage);
}
textAnalyticsDocuments.AddRange(textAnalyticsResponse.Documents);
}
if (sentimentSetting == SentimentAnalysisSetting.UtteranceLevel)
{
foreach (var document in textAnalyticsDocuments)
{
// Matching sentiment and transcription JSON by using the timestamp
var target = speechTranscript.RecognizedPhrases.Where(e => $"{e.Channel}_{e.Offset}".Equals(document.Id, StringComparison.Ordinal)).FirstOrDefault();
var nBest = target.NBest.FirstOrDefault();
if (nBest != null)
{
if (nBest.Sentiment == null)
{
nBest.Sentiment = new Sentiment();
}
nBest.Sentiment.Negative = document.ConfidenceScores.Negative;
nBest.Sentiment.Positive = document.ConfidenceScores.Positive;
nBest.Sentiment.Neutral = document.ConfidenceScores.Neutral;
}
}
}
else if (sentimentSetting == SentimentAnalysisSetting.AudioLevel)
{
foreach (var combinedRecognizedPhrase in speechTranscript.CombinedRecognizedPhrases)
{
var documents = textAnalyticsDocuments.Where(document => document.Id.StartsWith($"{combinedRecognizedPhrase.Channel}_", StringComparison.OrdinalIgnoreCase));
if (!documents.Any())
{
continue;
}
if (combinedRecognizedPhrase.Sentiment == null)
{
combinedRecognizedPhrase.Sentiment = new Sentiment();
}
combinedRecognizedPhrase.Sentiment.Negative = documents.Average(d => d.ConfidenceScores.Negative);
combinedRecognizedPhrase.Sentiment.Positive = documents.Average(d => d.ConfidenceScores.Positive);
combinedRecognizedPhrase.Sentiment.Neutral = documents.Average(d => d.ConfidenceScores.Neutral);
}
}
return(sentimentErrors);
}
public async Task <IEnumerable <string> > AddAudioLevelEntitiesAsync(
SpeechTranscript speechTranscript,
SentimentAnalysisSetting sentimentAnalysisSetting,
PiiRedactionSetting piiRedactionSetting)
{
speechTranscript = speechTranscript ?? throw new ArgumentNullException(nameof(speechTranscript));
var errors = new List <string>();
if (sentimentAnalysisSetting != SentimentAnalysisSetting.AudioLevel && piiRedactionSetting != PiiRedactionSetting.UtteranceAndAudioLevel)
{
return(errors);
}
// Remove other nBests if pii is redacted
if (piiRedactionSetting != PiiRedactionSetting.None)
{
speechTranscript.RecognizedPhrases.ToList().ForEach(phrase =>
{
if (phrase.NBest != null && phrase.NBest.Any())
{
var firstNBest = phrase.NBest.First();
phrase.NBest = new[] { firstNBest };
}
});
}
var documents = speechTranscript.CombinedRecognizedPhrases.Where(r => !string.IsNullOrEmpty(r.Display)).Select(r => new TextDocumentInput($"{r.Channel}", r.Display)
{
Language = Locale
});
var actions = new TextAnalyticsActions
{
DisplayName = "IngestionClient"
};
if (sentimentAnalysisSetting == SentimentAnalysisSetting.AudioLevel)
{
actions.AnalyzeSentimentActions = new List <AnalyzeSentimentAction>()
{
new AnalyzeSentimentAction()
};
}
if (piiRedactionSetting == PiiRedactionSetting.UtteranceAndAudioLevel)
{
var action = new RecognizePiiEntitiesAction();
if (!string.IsNullOrEmpty(FetchTranscriptionEnvironmentVariables.PiiCategories))
{
var piiEntityCategories = FetchTranscriptionEnvironmentVariables.PiiCategories.Split(",").Select(c => new PiiEntityCategory(c));
foreach (var category in piiEntityCategories)
{
action.CategoriesFilter.Add(category);
}
}
actions.RecognizePiiEntitiesActions = new List <RecognizePiiEntitiesAction>()
{
action
};
}
var(sentimentResults, piiResults, requestErrors) = await this.GetDocumentResultsAsync(documents, actions).ConfigureAwait(false);
errors.AddRange(requestErrors);
foreach (var combinedRecognizedPhrase in speechTranscript.CombinedRecognizedPhrases)
{
var channel = combinedRecognizedPhrase.Channel;
var sentimentResult = sentimentResults.Where(document => document.Id.Equals($"{channel}", StringComparison.OrdinalIgnoreCase)).SingleOrDefault();
if (sentimentResult != null)
{
combinedRecognizedPhrase.Sentiment = new Sentiment()
{
Negative = sentimentResult.DocumentSentiment.ConfidenceScores.Negative,
Positive = sentimentResult.DocumentSentiment.ConfidenceScores.Positive,
Neutral = sentimentResult.DocumentSentiment.ConfidenceScores.Neutral,
};
}
var piiResult = piiResults.Where(document => document.Id.Equals($"{channel}", StringComparison.OrdinalIgnoreCase)).SingleOrDefault();
if (piiResult != null)
{
var redactedText = piiResult.Entities.RedactedText;
combinedRecognizedPhrase.Display = redactedText;
combinedRecognizedPhrase.ITN = string.Empty;
combinedRecognizedPhrase.MaskedITN = string.Empty;
combinedRecognizedPhrase.Lexical = string.Empty;
var phrases = speechTranscript.RecognizedPhrases.Where(phrase => phrase.Channel == channel);
var startIndex = 0;
foreach (var phrase in phrases)
{
var firstNBest = phrase.NBest.FirstOrDefault();
if (firstNBest != null && !string.IsNullOrEmpty(firstNBest.Display))
{
firstNBest.Display = redactedText.Substring(startIndex, firstNBest.Display.Length);
firstNBest.ITN = string.Empty;
firstNBest.MaskedITN = string.Empty;
firstNBest.Lexical = string.Empty;
startIndex += firstNBest.Display.Length + 1;
}
}
}
}
return(errors);
}
