internal static async Task ProcessReportFileAsync(string reportFileContent, ILogger log)
{
var transcriptionReportFile = JsonConvert.DeserializeObject <TranscriptionReportFile>(reportFileContent);
var failedTranscriptions = transcriptionReportFile.Details.
Where(detail => !string.IsNullOrEmpty(detail.Status) &&
detail.Status.Equals("Failed", StringComparison.OrdinalIgnoreCase) &&
!string.IsNullOrEmpty(detail.SourceUrl));
foreach (var failedTranscription in failedTranscriptions)
{
var fileName = StorageUtilities.GetFileNameFromUri(new Uri(failedTranscription.SourceUrl));
var message = $"Transcription \"{fileName}\" failed with error \"{failedTranscription.ErrorKind}\" and message \"{failedTranscription.ErrorMessage}\"";
log.LogError(message);
var errorTxtname = fileName + ".txt";
await StorageUtilities.WriteTextFileToBlobAsync(
FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage,
message,
FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer,
errorTxtname,
log).ConfigureAwait(false);
}
}
public void GetContainerFromSasTest()
{
var containerName = StorageUtilities.GetContainerNameFromUri(new Uri(TestSasUri));
var fileName = StorageUtilities.GetFileNameFromUri(new Uri(TestSasUri));
var fileNameWithoutExtension = StorageUtilities.GetFileNameWithoutExtension(fileName);
Assert.AreEqual("testContainer", containerName);
Assert.AreEqual("testfolder/test.wav", fileName);
Assert.AreEqual("testfolder/test", fileNameWithoutExtension);
}
private async Task WriteFailedJobLogToStorageAsync(IEnumerable <ServiceBusMessage> serviceBusMessages, string errorMessage, string jobName)
{
var errorOutputContainer = StartTranscriptionEnvironmentVariables.ErrorReportOutputContainer;
var errorFileName = jobName + ".txt";
await StorageUtilities.WriteTextFileToBlobAsync(StartTranscriptionEnvironmentVariables.AzureWebJobsStorage, errorMessage, errorOutputContainer, errorFileName, Logger).ConfigureAwait(false);
foreach (var message in serviceBusMessages)
{
var fileName = StorageUtilities.GetFileNameFromUri(message.Data.Url);
errorFileName = fileName + ".txt";
await StorageUtilities.WriteTextFileToBlobAsync(StartTranscriptionEnvironmentVariables.AzureWebJobsStorage, errorMessage, errorOutputContainer, errorFileName, Logger).ConfigureAwait(false);
}
}
public async Task StartTranscriptionAsync(Message message)
{
if (message == null)
{
throw new ArgumentNullException(nameof(message));
}
var busMessage = JsonConvert.DeserializeObject <ServiceBusMessage>(Encoding.UTF8.GetString(message.Body));
var audioFileName = StorageUtilities.GetFileNameFromUri(busMessage.Data.Url);
var startDateTime = DateTime.UtcNow;
// Check if language identification is required:
var secondaryLocale = StartTranscriptionEnvironmentVariables.SecondaryLocale;
if (!string.IsNullOrEmpty(secondaryLocale) &&
!secondaryLocale.Equals("None", StringComparison.OrdinalIgnoreCase) &&
!secondaryLocale.Equals(Locale, StringComparison.OrdinalIgnoreCase))
{
secondaryLocale = secondaryLocale.Split('|')[0].Trim();
Logger.LogInformation($"Primary locale: {Locale}");
Logger.LogInformation($"Secondary locale: {secondaryLocale}");
var languageID = new LanguageIdentification(StartTranscriptionEnvironmentVariables.AzureSpeechServicesKey, StartTranscriptionEnvironmentVariables.AzureSpeechServicesRegion);
var fileExtension = Path.GetExtension(audioFileName);
var sasUrl = await StorageUtilities.CreateSASAsync(StartTranscriptionEnvironmentVariables.AzureWebJobsStorage, busMessage.Data.Url, Logger).ConfigureAwait(false);
var byteArray = await StorageUtilities.DownloadFileFromSAS(sasUrl).ConfigureAwait(false);
var identifiedLocale = await languageID.DetectLanguage(byteArray, fileExtension, Locale, secondaryLocale).ConfigureAwait(false);
Logger.LogInformation($"Identified locale: {identifiedLocale}");
Locale = identifiedLocale;
}
await StartBatchTranscriptionJobAsync(new[] { message }, audioFileName, startDateTime).ConfigureAwait(false);
}
internal static async Task ProcessSucceededTranscriptionAsync(BatchClient client, PostTranscriptionServiceBusMessage serviceBusMessage, Guid transcriptionId, string jobName, ILogger log)
{
log.LogInformation($"Got succeeded transcription for job {jobName}");
var jsonContainer = FetchTranscriptionEnvironmentVariables.JsonResultOutputContainer;
var transcriptionFiles = await client.GetTranscriptionFilesAsync(transcriptionId).ConfigureAwait(false);
var resultFiles = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.Transcription);
var containsMultipleTranscriptions = resultFiles.Skip(1).Any();
var textAnalyticsKey = FetchTranscriptionEnvironmentVariables.TextAnalyticsKey;
var textAnalyticsRegion = FetchTranscriptionEnvironmentVariables.TextAnalyticsRegion;
var textAnalyticsInfoProvided = !string.IsNullOrEmpty(textAnalyticsKey) &&
!string.IsNullOrEmpty(textAnalyticsRegion) &&
!textAnalyticsRegion.Equals("none", StringComparison.OrdinalIgnoreCase);
var textAnalytics = textAnalyticsInfoProvided ? new TextAnalytics(serviceBusMessage.Locale, textAnalyticsKey, textAnalyticsRegion, log) : null;
var generalErrorsStringBuilder = new StringBuilder();
foreach (var resultFile in resultFiles)
{
var fileName = string.Empty;
try
{
var transcriptionResultJson = string.Empty;
using (var webClient = new WebClient())
{
transcriptionResultJson = webClient.DownloadString(resultFile.Links.ContentUrl);
}
var transcriptionResult = JsonConvert.DeserializeObject <SpeechTranscript>(transcriptionResultJson);
fileName = StorageUtilities.GetFileNameFromUri(new Uri(transcriptionResult.Source));
log.LogInformation($"Filename is {fileName}");
if (transcriptionResult.RecognizedPhrases == null || transcriptionResult.RecognizedPhrases.All(phrase => !phrase.RecognitionStatus.Equals("Success", StringComparison.Ordinal)))
{
continue;
}
var textAnalyticsErrors = new List <string>();
if (serviceBusMessage.AddSentimentAnalysis)
{
var sentimentErrors = await textAnalytics.AddSentimentToTranscriptAsync(transcriptionResult).ConfigureAwait(false);
textAnalyticsErrors.AddRange(sentimentErrors);
}
if (serviceBusMessage.AddEntityRedaction)
{
var entityRedactionErrors = await textAnalytics.RedactEntitiesAsync(transcriptionResult).ConfigureAwait(false);
textAnalyticsErrors.AddRange(entityRedactionErrors);
}
var editedTranscriptionResultJson = JsonConvert.SerializeObject(transcriptionResult, Newtonsoft.Json.Formatting.Indented);
// Store transcript json:
var jsonFileName = $"{fileName}.json";
await StorageUtilities.WriteTextFileToBlobAsync(FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage, editedTranscriptionResultJson, jsonContainer, jsonFileName, log).ConfigureAwait(false);
if (FetchTranscriptionEnvironmentVariables.CreateHtmlResultFile)
{
var htmlContainer = FetchTranscriptionEnvironmentVariables.HtmlResultOutputContainer;
var htmlFileName = $"{fileName}.html";
var displayResults = TranscriptionToHtml.ToHTML(transcriptionResult, jobName);
await StorageUtilities.WriteTextFileToBlobAsync(FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage, displayResults, htmlContainer, htmlFileName, log).ConfigureAwait(false);
}
if (textAnalyticsErrors.Any())
{
var distinctErrors = textAnalyticsErrors.Distinct();
var errorMessage = $"File {(string.IsNullOrEmpty(fileName) ? "unknown" : fileName)}:\n{string.Join('\n', distinctErrors)}";
generalErrorsStringBuilder.AppendLine(errorMessage);
}
if (FetchTranscriptionEnvironmentVariables.UseSqlDatabase)
{
var duration = XmlConvert.ToTimeSpan(transcriptionResult.Duration);
var approximatedCost = CostEstimation.GetCostEstimation(
duration,
transcriptionResult.CombinedRecognizedPhrases.Count(),
serviceBusMessage.UsesCustomModel,
serviceBusMessage.AddSentimentAnalysis,
serviceBusMessage.AddEntityRedaction);
var dbConnectionString = FetchTranscriptionEnvironmentVariables.DatabaseConnectionString;
using var dbConnector = new DatabaseConnector(log, dbConnectionString);
await dbConnector.StoreTranscriptionAsync(
containsMultipleTranscriptions?Guid.NewGuid() : transcriptionId,
serviceBusMessage.Locale,
string.IsNullOrEmpty(fileName)?jobName : fileName,
(float)approximatedCost,
transcriptionResult).ConfigureAwait(false);
}
}
catch (Exception e)
{
if (string.IsNullOrEmpty(fileName) && e is ArgumentNullException)
{
var errorMessage = $"Transcription file name is unknown, failed with message: {e.Message}";
log.LogError(errorMessage);
generalErrorsStringBuilder.AppendLine(errorMessage);
continue;
}
else if (e is JsonException || e is SqlException)
{
var errorTxtname = fileName + ".txt";
var errorMessage = $"Transcription result processing failed with exception: {e.Message}";
log.LogError(errorMessage);
await StorageUtilities.WriteTextFileToBlobAsync(
FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage,
errorMessage,
FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer,
errorTxtname,
log).ConfigureAwait(false);
continue;
}
throw;
}
}
var errors = generalErrorsStringBuilder.ToString();
if (!string.IsNullOrEmpty(errors))
{
var errorTxtname = jobName + ".txt";
await StorageUtilities.WriteTextFileToBlobAsync(
FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage,
errors,
FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer,
errorTxtname,
log).ConfigureAwait(false);
}
// Delete trace from service
client.DeleteTranscriptionAsync(transcriptionId).ConfigureAwait(false).GetAwaiter().GetResult();
}