public async Task StartTranscriptionAsync(Message message) { if (message == null) { throw new ArgumentNullException(nameof(message)); } var busMessage = JsonConvert.DeserializeObject <ServiceBusMessage>(Encoding.UTF8.GetString(message.Body)); var audioFileName = StorageConnector.GetFileNameFromUri(busMessage.Data.Url); // Check if language identification is required: var secondaryLocale = StartTranscriptionEnvironmentVariables.SecondaryLocale; if (!string.IsNullOrEmpty(secondaryLocale) && !secondaryLocale.Equals("None", StringComparison.OrdinalIgnoreCase) && !secondaryLocale.Equals(Locale, StringComparison.OrdinalIgnoreCase)) { secondaryLocale = secondaryLocale.Split('|')[0].Trim(); Logger.LogInformation($"Primary locale: {Locale}"); Logger.LogInformation($"Secondary locale: {secondaryLocale}"); var languageID = new LanguageIdentification(SubscriptionKey, SubscriptionRegion); var fileExtension = Path.GetExtension(audioFileName); var sasUrl = StorageConnectorInstance.CreateSas(busMessage.Data.Url); var byteArray = await StorageConnector.DownloadFileFromSAS(sasUrl).ConfigureAwait(false); var identifiedLocale = await languageID.DetectLanguage(byteArray, fileExtension, Locale, secondaryLocale).ConfigureAwait(false); Logger.LogInformation($"Identified locale: {identifiedLocale}"); Locale = identifiedLocale; } await StartBatchTranscriptionJobAsync(new[] { message }, audioFileName).ConfigureAwait(false); }
private async Task RetryOrFailMessagesAsync(IEnumerable <Message> messages, string errorMessage) { foreach (var message in messages) { var sbMessage = JsonConvert.DeserializeObject <ServiceBusMessage>(Encoding.UTF8.GetString(message.Body)); if (sbMessage.RetryCount >= StartTranscriptionEnvironmentVariables.RetryLimit) { var fileName = StorageConnector.GetFileNameFromUri(sbMessage.Data.Url); var errorFileName = fileName + ".txt"; var retryExceededErrorMessage = $"Exceeded retry count for transcription {fileName} with error message {errorMessage}."; Logger.LogInformation(retryExceededErrorMessage); await StorageConnectorInstance.WriteTextFileToBlobAsync(retryExceededErrorMessage, ErrorReportContaineName, errorFileName, Logger).ConfigureAwait(false); await StorageConnectorInstance.MoveFileAsync( AudioInputContainerName, fileName, StartTranscriptionEnvironmentVariables.ErrorFilesOutputContainer, fileName, Logger).ConfigureAwait(false); } else { sbMessage.RetryCount += 1; var messageDelay = GetMessageDelayTime(sbMessage.RetryCount); var newMessage = new Message(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(sbMessage))); await ServiceBusUtilities.SendServiceBusMessageAsync(StartQueueClientInstance, newMessage, Logger, messageDelay).ConfigureAwait(false); } } }
private static async Task WriteFailedJobLogToStorageAsync(TranscriptionStartedMessage transcriptionStartedMessage, string errorMessage, string jobName, ILogger log) { var errorOutputContainer = FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer; var jobErrorFileName = $"jobs/{jobName}.txt"; await StorageConnectorInstance.WriteTextFileToBlobAsync(errorMessage, errorOutputContainer, jobErrorFileName, log).ConfigureAwait(false); foreach (var audioFileInfo in transcriptionStartedMessage.AudioFileInfos) { var fileName = StorageConnector.GetFileNameFromUri(new Uri(audioFileInfo.FileUrl)); var errorFileName = fileName + ".txt"; try { await StorageConnectorInstance.WriteTextFileToBlobAsync(errorMessage, errorOutputContainer, errorFileName, log).ConfigureAwait(false); await StorageConnectorInstance.MoveFileAsync( FetchTranscriptionEnvironmentVariables.AudioInputContainer, fileName, FetchTranscriptionEnvironmentVariables.ErrorFilesOutputContainer, fileName, log).ConfigureAwait(false); } catch (StorageException e) { log.LogError($"Storage Exception {e} while writing error log to file and moving result"); } } }
public bool IsValidServiceBusMessage(Message message) { if (message == null || message.Body == null || !message.Body.Any()) { Logger.LogError($"Message {nameof(message)} is null."); return(false); } var messageBody = Encoding.UTF8.GetString(message.Body); try { var serviceBusMessage = JsonConvert.DeserializeObject <ServiceBusMessage>(messageBody); if (serviceBusMessage.EventType.Contains("BlobCreate", StringComparison.OrdinalIgnoreCase) && StorageConnector.GetContainerNameFromUri(serviceBusMessage.Data.Url).Equals(AudioInputContainerName, StringComparison.Ordinal)) { return(true); } } catch (Exception e) { Logger.LogError($"Exception {e.Message} while parsing message {messageBody} - message will be ignored."); return(false); } return(false); }
public void GetContainerFromSasTest() { var containerName = StorageConnector.GetContainerNameFromUri(new Uri(TestSasUri)); var fileName = StorageConnector.GetFileNameFromUri(new Uri(TestSasUri)); var fileNameWithoutExtension = StorageConnector.GetFileNameWithoutExtension(fileName); Assert.AreEqual("testContainer", containerName); Assert.AreEqual("testfolder/test.wav", fileName); Assert.AreEqual("testfolder/test", fileNameWithoutExtension); }
private async Task WriteFailedJobLogToStorageAsync(IEnumerable <ServiceBusMessage> serviceBusMessages, string errorMessage, string jobName) { var jobErrorFileName = $"jobs/{jobName}.txt"; await StorageConnectorInstance.WriteTextFileToBlobAsync(errorMessage, ErrorReportContaineName, jobErrorFileName, Logger).ConfigureAwait(false); foreach (var message in serviceBusMessages) { var fileName = StorageConnector.GetFileNameFromUri(message.Data.Url); var errorFileName = fileName + ".txt"; await ProcessFailedFileAsync(fileName, errorMessage, errorFileName).ConfigureAwait(false); } }
private async Task WriteFailedJobLogToStorageAsync(IEnumerable <ServiceBusMessage> serviceBusMessages, string errorMessage, string jobName) { var jobErrorFileName = $"jobs/{jobName}.txt"; await StorageConnectorInstance.WriteTextFileToBlobAsync(errorMessage, ErrorReportContaineName, jobErrorFileName, Logger).ConfigureAwait(false); foreach (var message in serviceBusMessages) { var fileName = StorageConnector.GetFileNameFromUri(message.Data.Url); var errorFileName = fileName + ".txt"; await StorageConnectorInstance.WriteTextFileToBlobAsync(errorMessage, ErrorReportContaineName, errorFileName, Logger).ConfigureAwait(false); await StorageConnectorInstance.MoveFileAsync( AudioInputContainerName, fileName, StartTranscriptionEnvironmentVariables.ErrorFilesOutputContainer, fileName, Logger).ConfigureAwait(false); } }
public File ToFile(StorageConnector storage) { // Create file File file; using (var stream = FileData.OpenReadStream()) { file = storage.Create(Category.ToString().ToLower(), FileData.FileName, stream); } // Fill fields file.AltText = AltText; if (!string.IsNullOrEmpty(FileName)) { file.Name = FileName; } return(file); }
private static async Task ProcessReportFileAsync(TranscriptionReportFile transcriptionReportFile, ILogger log) { var failedTranscriptions = transcriptionReportFile.Details. Where(detail => !string.IsNullOrEmpty(detail.Status) && detail.Status.Equals("Failed", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(detail.Source)); foreach (var failedTranscription in failedTranscriptions) { if (string.IsNullOrEmpty(failedTranscription.Source)) { continue; } var safeErrorCode = failedTranscription.ErrorKind ?? "unknown"; var safeErrorMessage = failedTranscription.ErrorMessage ?? "unknown"; var fileName = StorageConnector.GetFileNameFromUri(new Uri(failedTranscription.Source)); var message = $"Transcription \"{fileName}\" failed with error \"{safeErrorCode}\" and message \"{safeErrorMessage}\""; log.LogError(message); var errorTxtname = fileName + ".txt"; await StorageConnectorInstance.WriteTextFileToBlobAsync( message, FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer, errorTxtname, log).ConfigureAwait(false); await StorageConnectorInstance.MoveFileAsync( FetchTranscriptionEnvironmentVariables.AudioInputContainer, fileName, FetchTranscriptionEnvironmentVariables.ErrorFilesOutputContainer, fileName, false, log).ConfigureAwait(false); } }
private static async Task ProcessSucceededTranscriptionAsync(string transcriptionLocation, string subscriptionKey, TranscriptionStartedMessage serviceBusMessage, string jobName, ILogger log) { log.LogInformation($"Got succeeded transcription for job {jobName}"); var jsonContainer = FetchTranscriptionEnvironmentVariables.JsonResultOutputContainer; var textAnalyticsKey = FetchTranscriptionEnvironmentVariables.TextAnalyticsKey; var textAnalyticsRegion = FetchTranscriptionEnvironmentVariables.TextAnalyticsRegion; var transcriptionFiles = await BatchClient.GetTranscriptionFilesAsync(transcriptionLocation, subscriptionKey, log).ConfigureAwait(false); log.LogInformation($"Received transcription files."); var resultFiles = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.Transcription); var containsMultipleTranscriptions = resultFiles.Skip(1).Any(); var textAnalyticsInfoProvided = !string.IsNullOrEmpty(textAnalyticsKey) && !string.IsNullOrEmpty(textAnalyticsRegion) && !textAnalyticsRegion.Equals("none", StringComparison.OrdinalIgnoreCase); var textAnalytics = textAnalyticsInfoProvided ? new TextAnalytics(serviceBusMessage.Locale, textAnalyticsKey, textAnalyticsRegion, log) : null; var generalErrorsStringBuilder = new StringBuilder(); foreach (var resultFile in resultFiles) { log.LogInformation($"Getting result for file {resultFile.Name}"); var transcriptionResult = await BatchClient.GetSpeechTranscriptFromSasAsync(resultFile.Links.ContentUrl, log).ConfigureAwait(false); if (string.IsNullOrEmpty(transcriptionResult.Source)) { var errorMessage = $"Transcription source is unknown, skipping evaluation."; log.LogError(errorMessage); generalErrorsStringBuilder.AppendLine(errorMessage); continue; } var fileName = StorageConnector.GetFileNameFromUri(new Uri(transcriptionResult.Source)); if (transcriptionResult.RecognizedPhrases != null && transcriptionResult.RecognizedPhrases.All(phrase => phrase.RecognitionStatus.Equals("Success", StringComparison.Ordinal))) { var textAnalyticsErrors = new List <string>(); if (FetchTranscriptionEnvironmentVariables.SentimentAnalysisSetting != SentimentAnalysisSetting.None) { var sentimentErrors = await textAnalytics.AddSentimentToTranscriptAsync(transcriptionResult, FetchTranscriptionEnvironmentVariables.SentimentAnalysisSetting).ConfigureAwait(false); textAnalyticsErrors.AddRange(sentimentErrors); } if (FetchTranscriptionEnvironmentVariables.EntityRedactionSetting != EntityRedactionSetting.None) { var entityRedactionErrors = await textAnalytics.RedactEntitiesAsync(transcriptionResult, FetchTranscriptionEnvironmentVariables.EntityRedactionSetting).ConfigureAwait(false); textAnalyticsErrors.AddRange(entityRedactionErrors); } if (textAnalyticsErrors.Any()) { var distinctErrors = textAnalyticsErrors.Distinct(); var errorMessage = $"File {(string.IsNullOrEmpty(fileName) ? "unknown" : fileName)}:\n{string.Join('\n', distinctErrors)}"; generalErrorsStringBuilder.AppendLine(errorMessage); } } var editedTranscriptionResultJson = JsonConvert.SerializeObject( transcriptionResult, Newtonsoft.Json.Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); var jsonFileName = $"{fileName}.json"; await StorageConnectorInstance.WriteTextFileToBlobAsync(editedTranscriptionResultJson, jsonContainer, jsonFileName, log).ConfigureAwait(false); if (FetchTranscriptionEnvironmentVariables.CreateHtmlResultFile) { var htmlContainer = FetchTranscriptionEnvironmentVariables.HtmlResultOutputContainer; var htmlFileName = $"{fileName}.html"; var displayResults = TranscriptionToHtml.ToHtml(transcriptionResult, jobName); await StorageConnectorInstance.WriteTextFileToBlobAsync(displayResults, htmlContainer, htmlFileName, log).ConfigureAwait(false); } if (FetchTranscriptionEnvironmentVariables.UseSqlDatabase) { var duration = XmlConvert.ToTimeSpan(transcriptionResult.Duration); var approximatedCost = CostEstimation.GetCostEstimation( duration, transcriptionResult.CombinedRecognizedPhrases.Count(), serviceBusMessage.UsesCustomModel, FetchTranscriptionEnvironmentVariables.SentimentAnalysisSetting, FetchTranscriptionEnvironmentVariables.EntityRedactionSetting); var jobId = containsMultipleTranscriptions ? Guid.NewGuid() : new Guid(transcriptionLocation.Split('/').LastOrDefault()); var dbConnectionString = FetchTranscriptionEnvironmentVariables.DatabaseConnectionString; using var dbConnector = new DatabaseConnector(log, dbConnectionString); await dbConnector.StoreTranscriptionAsync( jobId, serviceBusMessage.Locale, string.IsNullOrEmpty(fileName)?jobName : fileName, (float)approximatedCost, transcriptionResult).ConfigureAwait(false); } } var generalErrors = generalErrorsStringBuilder.ToString(); if (!string.IsNullOrEmpty(generalErrors)) { var errorTxtname = $"jobs/{jobName}.txt"; await StorageConnectorInstance.WriteTextFileToBlobAsync( generalErrors, FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer, errorTxtname, log).ConfigureAwait(false); } var reportFile = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.TranscriptionReport).FirstOrDefault(); var reportFileContent = await BatchClient.GetTranscriptionReportFileFromSasAsync(reportFile.Links.ContentUrl, log).ConfigureAwait(false); await ProcessReportFileAsync(reportFileContent, log).ConfigureAwait(false); BatchClient.DeleteTranscriptionAsync(transcriptionLocation, subscriptionKey, log).ConfigureAwait(false).GetAwaiter().GetResult(); }
private static async Task ProcessFailedTranscriptionAsync(string transcriptionLocation, string subscriptionKey, TranscriptionStartedMessage serviceBusMessage, Transcription transcription, string jobName, ILogger log) { var safeErrorCode = transcription?.Properties?.Error?.Code ?? "unknown"; var safeErrorMessage = transcription?.Properties?.Error?.Message ?? "unknown"; var logMessage = $"Got failed transcription for job {jobName} with error {safeErrorMessage} (Error code: {safeErrorCode})."; log.LogInformation(logMessage); var transcriptionFiles = await BatchClient.GetTranscriptionFilesAsync(transcriptionLocation, subscriptionKey, log).ConfigureAwait(false); var errorReportOutput = logMessage; var reportFile = transcriptionFiles.Values.Where(t => t.Kind == TranscriptionFileKind.TranscriptionReport).FirstOrDefault(); if (reportFile?.Links?.ContentUrl != null) { var reportFileContent = await BatchClient.GetTranscriptionReportFileFromSasAsync(reportFile.Links.ContentUrl, log).ConfigureAwait(false); errorReportOutput += $"\nReport file: \n {JsonConvert.SerializeObject(reportFileContent)}"; } var errorOutputContainer = FetchTranscriptionEnvironmentVariables.ErrorReportOutputContainer; await StorageConnectorInstance.WriteTextFileToBlobAsync(errorReportOutput, errorOutputContainer, $"jobs/{jobName}.txt", log).ConfigureAwait(false); var retryAudioFile = IsRetryableError(safeErrorCode); foreach (var audio in serviceBusMessage.AudioFileInfos) { var fileName = StorageConnector.GetFileNameFromUri(new Uri(audio.FileUrl)); if (retryAudioFile && audio.RetryCount < FetchTranscriptionEnvironmentVariables.RetryLimit) { log.LogInformation($"Retrying transcription with name {fileName} - retry count: {audio.RetryCount}"); var sbMessage = new ServiceBusMessage { Data = new Data { Url = new Uri(audio.FileUrl) }, EventType = "BlobCreated", RetryCount = audio.RetryCount + 1 }; var audioFileMessage = new Message(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(sbMessage))); await ServiceBusUtilities.SendServiceBusMessageAsync(StartQueueClientInstance, audioFileMessage, log, TimeSpan.FromMinutes(1)).ConfigureAwait(false); } else { var message = $"Failed transcription with name {fileName} in job {jobName} after {audio.RetryCount} retries with error: {safeErrorMessage} (Error: {safeErrorCode})."; await StorageConnectorInstance.WriteTextFileToBlobAsync(message, errorOutputContainer, $"{fileName}.txt", log).ConfigureAwait(false); await StorageConnectorInstance.MoveFileAsync( FetchTranscriptionEnvironmentVariables.AudioInputContainer, fileName, FetchTranscriptionEnvironmentVariables.ErrorFilesOutputContainer, fileName, log).ConfigureAwait(false); } } await BatchClient.DeleteTranscriptionAsync(transcriptionLocation, subscriptionKey, log).ConfigureAwait(false); }
public static IEnumerable <string> GetEmployees([HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest req, ILogger log, ExecutionContext context) { log.LogInformation("C# HTTP trigger function processed a request for employees."); return(StorageConnector.GetEmployees(GetConfig(log, context))); }
public HomeController(IConfiguration config) { Config = config; storageConnector = new StorageConnector(Config["PortfolioApp:Connections:PortfolioStorage"]); }