Exemple #1
0
        internal static async Task ProcessReportFileAsync(string reportFileContent, ILogger log)
        {
            var transcriptionReportFile = JsonConvert.DeserializeObject <TranscriptionReportFile>(reportFileContent);

            var failedTranscriptions = transcriptionReportFile.Details.
                                       Where(detail => !string.IsNullOrEmpty(detail.Status) &&
                                             detail.Status.Equals("Failed", StringComparison.OrdinalIgnoreCase) &&
                                             !string.IsNullOrEmpty(detail.SourceUrl));

            foreach (var failedTranscription in failedTranscriptions)
            {
                var fileName = StorageUtilities.GetFileNameFromUri(new Uri(failedTranscription.SourceUrl));

                var message = $"Transcription \"{fileName}\" failed with error \"{failedTranscription.ErrorKind}\" and message \"{failedTranscription.ErrorMessage}\"";
                log.LogError(message);

                var errorTxtname = fileName + ".txt";
                await StorageUtilities.WriteTextFileToBlobAsync(
                    FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage,
                    message,
                    FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer,
                    errorTxtname,
                    log).ConfigureAwait(false);
            }
        }
        public void GetContainerFromSasTest()
        {
            var containerName            = StorageUtilities.GetContainerNameFromUri(new Uri(TestSasUri));
            var fileName                 = StorageUtilities.GetFileNameFromUri(new Uri(TestSasUri));
            var fileNameWithoutExtension = StorageUtilities.GetFileNameWithoutExtension(fileName);

            Assert.AreEqual("testContainer", containerName);
            Assert.AreEqual("testfolder/test.wav", fileName);
            Assert.AreEqual("testfolder/test", fileNameWithoutExtension);
        }
        private async Task WriteFailedJobLogToStorageAsync(IEnumerable <ServiceBusMessage> serviceBusMessages, string errorMessage, string jobName)
        {
            var errorOutputContainer = StartTranscriptionEnvironmentVariables.ErrorReportOutputContainer;

            var errorFileName = jobName + ".txt";
            await StorageUtilities.WriteTextFileToBlobAsync(StartTranscriptionEnvironmentVariables.AzureWebJobsStorage, errorMessage, errorOutputContainer, errorFileName, Logger).ConfigureAwait(false);

            foreach (var message in serviceBusMessages)
            {
                var fileName = StorageUtilities.GetFileNameFromUri(message.Data.Url);
                errorFileName = fileName + ".txt";
                await StorageUtilities.WriteTextFileToBlobAsync(StartTranscriptionEnvironmentVariables.AzureWebJobsStorage, errorMessage, errorOutputContainer, errorFileName, Logger).ConfigureAwait(false);
            }
        }
        public async Task StartTranscriptionAsync(Message message)
        {
            if (message == null)
            {
                throw new ArgumentNullException(nameof(message));
            }

            var busMessage    = JsonConvert.DeserializeObject <ServiceBusMessage>(Encoding.UTF8.GetString(message.Body));
            var audioFileName = StorageUtilities.GetFileNameFromUri(busMessage.Data.Url);
            var startDateTime = DateTime.UtcNow;

            // Check if language identification is required:
            var secondaryLocale = StartTranscriptionEnvironmentVariables.SecondaryLocale;

            if (!string.IsNullOrEmpty(secondaryLocale) &&
                !secondaryLocale.Equals("None", StringComparison.OrdinalIgnoreCase) &&
                !secondaryLocale.Equals(Locale, StringComparison.OrdinalIgnoreCase))
            {
                secondaryLocale = secondaryLocale.Split('|')[0].Trim();

                Logger.LogInformation($"Primary locale: {Locale}");
                Logger.LogInformation($"Secondary locale: {secondaryLocale}");

                var languageID    = new LanguageIdentification(StartTranscriptionEnvironmentVariables.AzureSpeechServicesKey, StartTranscriptionEnvironmentVariables.AzureSpeechServicesRegion);
                var fileExtension = Path.GetExtension(audioFileName);
                var sasUrl        = await StorageUtilities.CreateSASAsync(StartTranscriptionEnvironmentVariables.AzureWebJobsStorage, busMessage.Data.Url, Logger).ConfigureAwait(false);

                var byteArray = await StorageUtilities.DownloadFileFromSAS(sasUrl).ConfigureAwait(false);

                var identifiedLocale = await languageID.DetectLanguage(byteArray, fileExtension, Locale, secondaryLocale).ConfigureAwait(false);

                Logger.LogInformation($"Identified locale: {identifiedLocale}");
                Locale = identifiedLocale;
            }

            await StartBatchTranscriptionJobAsync(new[] { message }, audioFileName, startDateTime).ConfigureAwait(false);
        }
Exemple #5
0
        internal static async Task ProcessSucceededTranscriptionAsync(BatchClient client, PostTranscriptionServiceBusMessage serviceBusMessage, Guid transcriptionId, string jobName, ILogger log)
        {
            log.LogInformation($"Got succeeded transcription for job {jobName}");

            var jsonContainer = FetchTranscriptionEnvironmentVariables.JsonResultOutputContainer;

            var transcriptionFiles = await client.GetTranscriptionFilesAsync(transcriptionId).ConfigureAwait(false);

            var resultFiles = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.Transcription);
            var containsMultipleTranscriptions = resultFiles.Skip(1).Any();

            var textAnalyticsKey    = FetchTranscriptionEnvironmentVariables.TextAnalyticsKey;
            var textAnalyticsRegion = FetchTranscriptionEnvironmentVariables.TextAnalyticsRegion;

            var textAnalyticsInfoProvided = !string.IsNullOrEmpty(textAnalyticsKey) &&
                                            !string.IsNullOrEmpty(textAnalyticsRegion) &&
                                            !textAnalyticsRegion.Equals("none", StringComparison.OrdinalIgnoreCase);

            var textAnalytics = textAnalyticsInfoProvided ? new TextAnalytics(serviceBusMessage.Locale, textAnalyticsKey, textAnalyticsRegion, log) : null;

            var generalErrorsStringBuilder = new StringBuilder();

            foreach (var resultFile in resultFiles)
            {
                var fileName = string.Empty;

                try
                {
                    var transcriptionResultJson = string.Empty;
                    using (var webClient = new WebClient())
                    {
                        transcriptionResultJson = webClient.DownloadString(resultFile.Links.ContentUrl);
                    }

                    var transcriptionResult = JsonConvert.DeserializeObject <SpeechTranscript>(transcriptionResultJson);
                    fileName = StorageUtilities.GetFileNameFromUri(new Uri(transcriptionResult.Source));
                    log.LogInformation($"Filename is {fileName}");

                    if (transcriptionResult.RecognizedPhrases == null || transcriptionResult.RecognizedPhrases.All(phrase => !phrase.RecognitionStatus.Equals("Success", StringComparison.Ordinal)))
                    {
                        continue;
                    }

                    var textAnalyticsErrors = new List <string>();

                    if (serviceBusMessage.AddSentimentAnalysis)
                    {
                        var sentimentErrors = await textAnalytics.AddSentimentToTranscriptAsync(transcriptionResult).ConfigureAwait(false);

                        textAnalyticsErrors.AddRange(sentimentErrors);
                    }

                    if (serviceBusMessage.AddEntityRedaction)
                    {
                        var entityRedactionErrors = await textAnalytics.RedactEntitiesAsync(transcriptionResult).ConfigureAwait(false);

                        textAnalyticsErrors.AddRange(entityRedactionErrors);
                    }

                    var editedTranscriptionResultJson = JsonConvert.SerializeObject(transcriptionResult, Newtonsoft.Json.Formatting.Indented);

                    // Store transcript json:
                    var jsonFileName = $"{fileName}.json";
                    await StorageUtilities.WriteTextFileToBlobAsync(FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage, editedTranscriptionResultJson, jsonContainer, jsonFileName, log).ConfigureAwait(false);

                    if (FetchTranscriptionEnvironmentVariables.CreateHtmlResultFile)
                    {
                        var htmlContainer  = FetchTranscriptionEnvironmentVariables.HtmlResultOutputContainer;
                        var htmlFileName   = $"{fileName}.html";
                        var displayResults = TranscriptionToHtml.ToHTML(transcriptionResult, jobName);
                        await StorageUtilities.WriteTextFileToBlobAsync(FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage, displayResults, htmlContainer, htmlFileName, log).ConfigureAwait(false);
                    }

                    if (textAnalyticsErrors.Any())
                    {
                        var distinctErrors = textAnalyticsErrors.Distinct();
                        var errorMessage   = $"File {(string.IsNullOrEmpty(fileName) ? "unknown" : fileName)}:\n{string.Join('\n', distinctErrors)}";

                        generalErrorsStringBuilder.AppendLine(errorMessage);
                    }

                    if (FetchTranscriptionEnvironmentVariables.UseSqlDatabase)
                    {
                        var duration         = XmlConvert.ToTimeSpan(transcriptionResult.Duration);
                        var approximatedCost = CostEstimation.GetCostEstimation(
                            duration,
                            transcriptionResult.CombinedRecognizedPhrases.Count(),
                            serviceBusMessage.UsesCustomModel,
                            serviceBusMessage.AddSentimentAnalysis,
                            serviceBusMessage.AddEntityRedaction);

                        var dbConnectionString = FetchTranscriptionEnvironmentVariables.DatabaseConnectionString;
                        using var dbConnector = new DatabaseConnector(log, dbConnectionString);
                        await dbConnector.StoreTranscriptionAsync(
                            containsMultipleTranscriptions?Guid.NewGuid() : transcriptionId,
                                serviceBusMessage.Locale,
                                string.IsNullOrEmpty(fileName)?jobName : fileName,
                                (float)approximatedCost,
                                transcriptionResult).ConfigureAwait(false);
                    }
                }
                catch (Exception e)
                {
                    if (string.IsNullOrEmpty(fileName) && e is ArgumentNullException)
                    {
                        var errorMessage = $"Transcription file name is unknown, failed with message: {e.Message}";
                        log.LogError(errorMessage);

                        generalErrorsStringBuilder.AppendLine(errorMessage);

                        continue;
                    }
                    else if (e is JsonException || e is SqlException)
                    {
                        var errorTxtname = fileName + ".txt";
                        var errorMessage = $"Transcription result processing failed with exception: {e.Message}";
                        log.LogError(errorMessage);

                        await StorageUtilities.WriteTextFileToBlobAsync(
                            FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage,
                            errorMessage,
                            FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer,
                            errorTxtname,
                            log).ConfigureAwait(false);

                        continue;
                    }

                    throw;
                }
            }

            var errors = generalErrorsStringBuilder.ToString();

            if (!string.IsNullOrEmpty(errors))
            {
                var errorTxtname = jobName + ".txt";

                await StorageUtilities.WriteTextFileToBlobAsync(
                    FetchTranscriptionEnvironmentVariables.AzureWebJobsStorage,
                    errors,
                    FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer,
                    errorTxtname,
                    log).ConfigureAwait(false);
            }

            // Delete trace from service
            client.DeleteTranscriptionAsync(transcriptionId).ConfigureAwait(false).GetAwaiter().GetResult();
        }