public async Task AnalyzeOperationRecognizePiiEntitiesWithPiiOptions()
{
var mockResponse = new MockResponse(202);
mockResponse.AddHeader(new HttpHeader("Operation-Location", "something/jobs/2a96a91f-7edf-4931-a880-3fdee1d56f15"));
var mockTransport = new MockTransport(new[] { mockResponse, mockResponse });
var client = CreateTestClient(mockTransport);
var documents = new List <string>
{
"Elon Musk is the CEO of SpaceX and Tesla."
};
var options = new RecognizePiiEntitiesOptions();
var actions = new RecognizePiiEntitiesAction(options);
TextAnalyticsActions batchActions = new TextAnalyticsActions()
{
RecognizePiiEntitiesActions = new List <RecognizePiiEntitiesAction>()
{
actions
},
};
await client.StartAnalyzeActionsAsync(documents, batchActions);
var contentString = GetString(mockTransport.Requests.Single().Content);
ValidateRequestOptions(contentString);
Assert.AreEqual(-1, contentString.IndexOf("domain"));
Assert.AreEqual(-1, contentString.IndexOf("piiCategories"));
}
/// <summary>
/// Submits text analytics requests depending on the settings (sentimentAnalysisSetting). The whole transcript (per channel) will be submitted in a single request.
/// (This means for instance that one single sentiment score will be generated per channel).
/// </summary>
/// <param name="speechTranscript">The speech transcript object.</param>
/// <param name="sentimentAnalysisSetting">The sentiment analysis setting.</param>
/// <param name="piiRedactionSetting">The PII redaction setting.</param>
/// <returns>The job ids and errors, if any were found.</returns>
public async Task <(IEnumerable <string> jobIds, IEnumerable <string> errors)> SubmitAudioLevelRequests(
SpeechTranscript speechTranscript,
SentimentAnalysisSetting sentimentAnalysisSetting,
PiiRedactionSetting piiRedactionSetting)
{
speechTranscript = speechTranscript ?? throw new ArgumentNullException(nameof(speechTranscript));
if (sentimentAnalysisSetting != SentimentAnalysisSetting.AudioLevel && piiRedactionSetting != PiiRedactionSetting.UtteranceAndAudioLevel)
{
return(new List <string>(), new List <string>());
}
var documents = speechTranscript.CombinedRecognizedPhrases.Where(r => !string.IsNullOrEmpty(r.Display)).Select(r => new TextDocumentInput($"{r.Channel}", r.Display)
{
Language = Locale
});
var actions = new TextAnalyticsActions
{
DisplayName = "IngestionClient"
};
if (sentimentAnalysisSetting == SentimentAnalysisSetting.AudioLevel)
{
actions.AnalyzeSentimentActions = new List <AnalyzeSentimentAction>()
{
new AnalyzeSentimentAction()
};
}
if (piiRedactionSetting == PiiRedactionSetting.UtteranceAndAudioLevel)
{
var action = new RecognizePiiEntitiesAction();
if (!string.IsNullOrEmpty(FetchTranscriptionEnvironmentVariables.PiiCategories))
{
var piiEntityCategories = FetchTranscriptionEnvironmentVariables.PiiCategories.Split(",").Select(c => new PiiEntityCategory(c));
foreach (var category in piiEntityCategories)
{
action.CategoriesFilter.Add(category);
}
}
actions.RecognizePiiEntitiesActions = new List <RecognizePiiEntitiesAction>()
{
action
};
}
return(await SubmitDocumentsAsync(documents, actions).ConfigureAwait(false));
}
public async Task AnalyzeOperationRecognizePiiEntitiesWithPiiOptionsFull()
{
var mockResponse = new MockResponse(202);
mockResponse.AddHeader(new HttpHeader("Operation-Location", "something/jobs/2a96a91f-7edf-4931-a880-3fdee1d56f15"));
var mockTransport = new MockTransport(new[] { mockResponse, mockResponse });
var client = CreateTestClient(mockTransport);
var documents = new List <string>
{
"Elon Musk is the CEO of SpaceX and Tesla."
};
var options = new RecognizePiiEntitiesOptions()
{
ModelVersion = "latest",
DisableServiceLogs = true,
IncludeStatistics = true,
DomainFilter = PiiEntityDomain.ProtectedHealthInformation,
CategoriesFilter = { PiiEntityCategory.USSocialSecurityNumber }
};
var actions = new RecognizePiiEntitiesAction(options);
TextAnalyticsActions batchActions = new TextAnalyticsActions()
{
RecognizePiiEntitiesActions = new List <RecognizePiiEntitiesAction>()
{
actions
},
};
await client.StartAnalyzeActionsAsync(documents, batchActions);
var contentString = GetString(mockTransport.Requests.Single().Content);
ValidateRequestOptions(contentString, true);
string domaintFilter = contentString.Substring(contentString.IndexOf("domain"), 13);
var expectedDomainFilterContent = "domain\":\"phi\"";
Assert.AreEqual(expectedDomainFilterContent, domaintFilter);
string piiCategories = contentString.Substring(contentString.IndexOf("piiCategories"), 41);
var expectedPiiCategoriesContent = "piiCategories\":[\"USSocialSecurityNumber\"]";
Assert.AreEqual(expectedPiiCategoriesContent, piiCategories);
}
public async Task AnalyzeOperationRecognizePiiEntitiesWithDisableServiceLogs()
{
var mockResponse = new MockResponse(202);
mockResponse.AddHeader(new HttpHeader("Operation-Location", "something/jobs/2a96a91f-7edf-4931-a880-3fdee1d56f15"));
var mockTransport = new MockTransport(new[] { mockResponse, mockResponse });
var client = CreateTestClient(mockTransport);
var documents = new List <string>
{
"Elon Musk is the CEO of SpaceX and Tesla."
};
var actions = new RecognizePiiEntitiesAction()
{
DisableServiceLogs = true
};
TextAnalyticsActions batchActions = new TextAnalyticsActions()
{
RecognizePiiEntitiesActions = new List <RecognizePiiEntitiesAction>()
{
actions
},
};
await client.StartAnalyzeActionsAsync(documents, batchActions);
var content = mockTransport.Requests.Single().Content;
using var stream = new MemoryStream();
await content.WriteToAsync(stream, default);
stream.Position = 0;
using var streamReader = new StreamReader(stream);
string contentString = streamReader.ReadToEnd();
string logging = contentString.Substring(contentString.IndexOf("loggingOptOut"), 19);
var expectedContent = "loggingOptOut\":true";
Assert.AreEqual(expectedContent, logging);
}
public async Task <IEnumerable <string> > AddAudioLevelEntitiesAsync(
SpeechTranscript speechTranscript,
SentimentAnalysisSetting sentimentAnalysisSetting,
PiiRedactionSetting piiRedactionSetting)
{
speechTranscript = speechTranscript ?? throw new ArgumentNullException(nameof(speechTranscript));
var errors = new List <string>();
if (sentimentAnalysisSetting != SentimentAnalysisSetting.AudioLevel && piiRedactionSetting != PiiRedactionSetting.UtteranceAndAudioLevel)
{
return(errors);
}
// Remove other nBests if pii is redacted
if (piiRedactionSetting != PiiRedactionSetting.None)
{
speechTranscript.RecognizedPhrases.ToList().ForEach(phrase =>
{
if (phrase.NBest != null && phrase.NBest.Any())
{
var firstNBest = phrase.NBest.First();
phrase.NBest = new[] { firstNBest };
}
});
}
var documents = speechTranscript.CombinedRecognizedPhrases.Where(r => !string.IsNullOrEmpty(r.Display)).Select(r => new TextDocumentInput($"{r.Channel}", r.Display)
{
Language = Locale
});
var actions = new TextAnalyticsActions
{
DisplayName = "IngestionClient"
};
if (sentimentAnalysisSetting == SentimentAnalysisSetting.AudioLevel)
{
actions.AnalyzeSentimentActions = new List <AnalyzeSentimentAction>()
{
new AnalyzeSentimentAction()
};
}
if (piiRedactionSetting == PiiRedactionSetting.UtteranceAndAudioLevel)
{
var action = new RecognizePiiEntitiesAction();
if (!string.IsNullOrEmpty(FetchTranscriptionEnvironmentVariables.PiiCategories))
{
var piiEntityCategories = FetchTranscriptionEnvironmentVariables.PiiCategories.Split(",").Select(c => new PiiEntityCategory(c));
foreach (var category in piiEntityCategories)
{
action.CategoriesFilter.Add(category);
}
}
actions.RecognizePiiEntitiesActions = new List <RecognizePiiEntitiesAction>()
{
action
};
}
var(sentimentResults, piiResults, requestErrors) = await this.GetDocumentResultsAsync(documents, actions).ConfigureAwait(false);
errors.AddRange(requestErrors);
foreach (var combinedRecognizedPhrase in speechTranscript.CombinedRecognizedPhrases)
{
var channel = combinedRecognizedPhrase.Channel;
var sentimentResult = sentimentResults.Where(document => document.Id.Equals($"{channel}", StringComparison.OrdinalIgnoreCase)).SingleOrDefault();
if (sentimentResult != null)
{
combinedRecognizedPhrase.Sentiment = new Sentiment()
{
Negative = sentimentResult.DocumentSentiment.ConfidenceScores.Negative,
Positive = sentimentResult.DocumentSentiment.ConfidenceScores.Positive,
Neutral = sentimentResult.DocumentSentiment.ConfidenceScores.Neutral,
};
}
var piiResult = piiResults.Where(document => document.Id.Equals($"{channel}", StringComparison.OrdinalIgnoreCase)).SingleOrDefault();
if (piiResult != null)
{
var redactedText = piiResult.Entities.RedactedText;
combinedRecognizedPhrase.Display = redactedText;
combinedRecognizedPhrase.ITN = string.Empty;
combinedRecognizedPhrase.MaskedITN = string.Empty;
combinedRecognizedPhrase.Lexical = string.Empty;
var phrases = speechTranscript.RecognizedPhrases.Where(phrase => phrase.Channel == channel);
var startIndex = 0;
foreach (var phrase in phrases)
{
var firstNBest = phrase.NBest.FirstOrDefault();
if (firstNBest != null && !string.IsNullOrEmpty(firstNBest.Display))
{
firstNBest.Display = redactedText.Substring(startIndex, firstNBest.Display.Length);
firstNBest.ITN = string.Empty;
firstNBest.MaskedITN = string.Empty;
firstNBest.Lexical = string.Empty;
startIndex += firstNBest.Display.Length + 1;
}
}
}
}
return(errors);
}
