public async Task ExecuteAsync(WorkerJobHelper <AIJob> jobHelper)
{
S3Locator inputFile;
if (!jobHelper.JobInput.TryGet <S3Locator>(nameof(inputFile), out inputFile))
{
throw new Exception("Invalid or missing input file.");
}
string mediaFileUrl;
if (!string.IsNullOrWhiteSpace(inputFile.HttpEndpoint))
{
mediaFileUrl = inputFile.HttpEndpoint;
}
else
{
var bucketLocation = await inputFile.GetBucketLocationAsync();
var s3SubDomain = !string.IsNullOrWhiteSpace(bucketLocation) ? $"s3-{bucketLocation}" : "s3";
mediaFileUrl = $"https://{s3SubDomain}.amazonaws.com/{inputFile.AwsS3Bucket}/{inputFile.AwsS3Key}";
}
string mediaFormat;
if (mediaFileUrl.EndsWith("mp3", StringComparison.OrdinalIgnoreCase))
{
mediaFormat = "mp3";
}
else if (mediaFileUrl.EndsWith("mp4", StringComparison.OrdinalIgnoreCase))
{
mediaFormat = "mp4";
}
else if (mediaFileUrl.EndsWith("wav", StringComparison.OrdinalIgnoreCase))
{
mediaFormat = "wav";
}
else if (mediaFileUrl.EndsWith("flac", StringComparison.OrdinalIgnoreCase))
{
mediaFormat = "flac";
}
else
{
throw new Exception($"Unable to determine media format from input file '{mediaFileUrl}'");
}
var transcribeParameters = new StartTranscriptionJobRequest
{
TranscriptionJobName = "TranscriptionJob-" + jobHelper.JobAssignmentId.Substring(jobHelper.JobAssignmentId.LastIndexOf("/") + 1),
LanguageCode = "en-US",
Media = new Media {
MediaFileUri = mediaFileUrl
},
MediaFormat = mediaFormat,
OutputBucketName = jobHelper.Request.GetRequiredContextVariable("ServiceOutputBucket")
};
using (var transcribeService = new AmazonTranscribeServiceClient())
await transcribeService.StartTranscriptionJobAsync(transcribeParameters);
}
public override async Task <FunctionResponse> ProcessMessageAsync(S3Event request)
{
foreach (var record in request.Records)
{
LogInfo($"processing 's3://{record.S3.Bucket.Name}/{record.S3.Object.Key}'");
// extract the video etag from the filename
var videoEtag = Path.GetFileNameWithoutExtension(record.S3.Object.Key);
try {
// kick off a transcribe job
var transcriptionResponse = await _transcribe.StartTranscriptionJobAsync(
new StartTranscriptionJobRequest {
// TODO: need a better name (add timestamp?) so that we can re-transcribe a transcribed video
TranscriptionJobName = $"transcribe-{videoEtag}",
Media = new Media {
MediaFileUri = $"https://s3-{record.AwsRegion}.amazonaws.com/{record.S3.Bucket.Name}/{record.S3.Object.Key}"
},
MediaFormat = MediaFormat.Mp3,
LanguageCode = LanguageCode.EnUS,
OutputBucketName = _outputBucketName
},
CancellationToken.None
);
// send a message to the progress queue
await _sqsClient.SendMessageAsync(new Amazon.SQS.Model.SendMessageRequest {
MessageBody = SerializeJson(new ProgressMessage {
Type = MessageType.INDEXING_STARTED,
Message = SerializeJson(new IndexingStartedMessage {
Job = transcriptionResponse
})
}),
QueueUrl = _notifyQueueUrl
});
} catch (Exception e) {
LogError(e);
}
}
return(new FunctionResponse {
});
}
internal static async Task ProcessJobAssignmentAsync(AwsAiServiceWorkerRequest @event)
{
var resourceManager = @event.GetAwsV4ResourceManager();
var table = new DynamoDbTable(@event.StageVariables["TableName"]);
var jobAssignmentId = @event.JobAssignmentId;
try
{
// 1. Setting job assignment status to RUNNING
await UpdateJobAssignmentStatusAsync(resourceManager, table, jobAssignmentId, "RUNNING", null);
// 2. Retrieving WorkflowJob
var job = await RetrieveJobAsync(resourceManager, table, jobAssignmentId);
// 3. Retrieve JobProfile
var jobProfile = await RetrieveJobProfileAsync(resourceManager, job);
// 4. Retrieve job inputParameters
var jobInput = job.JobInput;
// 5. Check if we support jobProfile and if we have required parameters in jobInput
ValidateJobProfile(jobProfile, jobInput);
S3Locator inputFile;
if (!jobInput.TryGet <S3Locator>(nameof(inputFile), out inputFile))
{
throw new Exception("Invalid or missing input file.");
}
S3Locator outputLocation;
if (!jobInput.TryGet <S3Locator>(nameof(outputLocation), out outputLocation))
{
throw new Exception("Invalid or missing output location.");
}
switch (jobProfile.Name)
{
case JOB_PROFILE_TRANSCRIBE_AUDIO:
string mediaFileUrl;
if (!string.IsNullOrWhiteSpace(inputFile.HttpEndpoint))
{
mediaFileUrl = inputFile.HttpEndpoint;
}
else
{
var bucketLocation = await inputFile.GetBucketLocationAsync();
var s3SubDomain = !string.IsNullOrWhiteSpace(bucketLocation) ? $"s3-{bucketLocation}" : "s3";
mediaFileUrl = $"https://{s3SubDomain}.amazonaws.com/{inputFile.AwsS3Bucket}/{inputFile.AwsS3Key}";
}
string mediaFormat;
if (mediaFileUrl.EndsWith("mp3", StringComparison.OrdinalIgnoreCase))
{
mediaFormat = "mp3";
}
else if (mediaFileUrl.EndsWith("mp4", StringComparison.OrdinalIgnoreCase))
{
mediaFormat = "mp4";
}
else if (mediaFileUrl.EndsWith("wav", StringComparison.OrdinalIgnoreCase))
{
mediaFormat = "wav";
}
else if (mediaFileUrl.EndsWith("flac", StringComparison.OrdinalIgnoreCase))
{
mediaFormat = "flac";
}
else
{
throw new Exception($"Unable to determine media format from input file '{mediaFileUrl}'");
}
var transcribeParameters = new StartTranscriptionJobRequest
{
TranscriptionJobName = "TranscriptionJob-" + jobAssignmentId.Substring(jobAssignmentId.LastIndexOf("/") + 1),
LanguageCode = "en-US",
Media = new Media {
MediaFileUri = mediaFileUrl
},
MediaFormat = mediaFormat,
OutputBucketName = @event.StageVariables["ServiceOutputBucket"]
};
var transcribeService = new AmazonTranscribeServiceClient();
var startJobResponse = await transcribeService.StartTranscriptionJobAsync(transcribeParameters);
Logger.Debug(startJobResponse.ToMcmaJson().ToString());
break;
case JOB_PROFILE_TRANSLATE_TEXT:
var s3Bucket = inputFile.AwsS3Bucket;
var s3Key = inputFile.AwsS3Key;
GetObjectResponse s3Object;
try
{
s3Object = await inputFile.GetAsync();
}
catch (Exception error)
{
throw new Exception($"Unable to read file in bucket '{s3Bucket}' with key '{s3Key}'.", error);
}
var inputText = await new StreamReader(s3Object.ResponseStream).ReadToEndAsync();
var translateParameters = new TranslateTextRequest
{
SourceLanguageCode = jobInput.TryGet("sourceLanguageCode", out string srcLanguageCode) ? srcLanguageCode : "auto",
TargetLanguageCode = jobInput.Get <string>("targetLanguageCode"),
Text = inputText
};
var translateService = new AmazonTranslateClient();
var translateResponse = await translateService.TranslateTextAsync(translateParameters);
var s3Params = new PutObjectRequest
{
BucketName = outputLocation.AwsS3Bucket,
Key = (!string.IsNullOrWhiteSpace(outputLocation.AwsS3KeyPrefix) ? outputLocation.AwsS3Key : string.Empty) + Guid.NewGuid() + ".txt",
ContentBody = translateResponse.TranslatedText
};
var outputS3 = await outputLocation.GetClientAsync();
await outputS3.PutObjectAsync(s3Params);
var jobOutput = new JobParameterBag();
jobOutput["outputFile"] = new S3Locator
{
AwsS3Bucket = s3Params.BucketName,
AwsS3Key = s3Params.Key
};
Logger.Debug("Updating job assignment");
await UpdateJobAssignmentWithOutputAsync(table, jobAssignmentId, jobOutput);
await UpdateJobAssignmentStatusAsync(resourceManager, table, jobAssignmentId, "COMPLETED");
break;
case JOB_PROFILE_DETECT_CELEBRITIES:
var randomBytes = new byte[16];
new Random().NextBytes(randomBytes);
var clientToken = randomBytes.HexEncode();
var base64JobId = Encoding.UTF8.GetBytes(jobAssignmentId).HexEncode();
var rekoParams = new StartCelebrityRecognitionRequest
{
Video = new Video
{
S3Object = new Amazon.Rekognition.Model.S3Object
{
Bucket = inputFile.AwsS3Bucket,
Name = inputFile.AwsS3Key
}
},
ClientRequestToken = clientToken,
JobTag = base64JobId,
NotificationChannel = new NotificationChannel
{
RoleArn = REKO_SNS_ROLE_ARN,
SNSTopicArn = SNS_TOPIC_ARN
}
};
var rekognitionClient = new AmazonRekognitionClient();
var startCelebRecognitionResponse = await rekognitionClient.StartCelebrityRecognitionAsync(rekoParams);
Logger.Debug(startCelebRecognitionResponse.ToMcmaJson().ToString());
break;
}
}
catch (Exception ex)
{
Logger.Exception(ex);
try
{
await UpdateJobAssignmentStatusAsync(resourceManager, table, jobAssignmentId, "FAILED", ex.ToString());
}
catch (Exception innerEx)
{
Logger.Exception(innerEx);
}
}
}
private async void BeginTranscription_Click(object sender, RoutedEventArgs e)
{
this._ctlBeginTranscription.IsEnabled = false;
try
{
this._ctlStatusLog.Text = "";
this._ctlTranscription.Text = "";
var filepath = this._ctlFilepath.Text;
if (string.IsNullOrEmpty(filepath))
{
MessageBox.Show($"An mp4 file must be selected first before transcribing", "Error", MessageBoxButton.OK, MessageBoxImage.Error);
return;
}
if (!System.IO.File.Exists(filepath))
{
MessageBox.Show($"File {filepath} does not exist", "Error", MessageBoxButton.OK, MessageBoxImage.Error);
return;
}
var s3Key = System.IO.Path.GetFileName(filepath);
var s3Bucket = _ctlS3Bucket.Text;
var chain = new CredentialProfileStoreChain();
AWSCredentials credentials;
if (!chain.TryGetAWSCredentials(this._ctlProfile.Text, out credentials))
{
MessageBox.Show($"Profile {this._ctlProfile.Text} was not found", "Error", MessageBoxButton.OK, MessageBoxImage.Error);
return;
}
var region = RegionEndpoint.GetBySystemName(this._ctlRegion.Text);
var transcriptionJobName = $"{s3Key}-{Guid.NewGuid().ToString()}";
using (var s3Client = new AmazonS3Client(credentials, region))
using (var transcribeClient = new AmazonTranscribeServiceClient(credentials, region))
using (var httpClient = new HttpClient()) // Http Client to download the transcription once complete
{
AppendStatusLine("Ensuring S3 bucket exists");
await s3Client.PutBucketAsync(s3Bucket);
var transferUtility = new TransferUtility(s3Client);
AppendStatusLine("Starting upload");
var uploadRequest = new TransferUtilityUploadRequest
{
FilePath = filepath,
BucketName = s3Bucket,
Key = s3Key
};
uploadRequest.UploadProgressEvent += ProgressUploadStatus;
await transferUtility.UploadAsync(uploadRequest);
var mediaFileUri = $"https://s3.{region.SystemName}.amazonaws.com/{s3Bucket}/{s3Key}";
AppendStatusLine($"Upload Complete to: {mediaFileUri}");
await transcribeClient.StartTranscriptionJobAsync(new StartTranscriptionJobRequest
{
LanguageCode = LanguageCode.EnUS,
Media = new Media
{
MediaFileUri = mediaFileUri
},
MediaFormat = MediaFormat.Mp4,
TranscriptionJobName = transcriptionJobName
});
AppendStatusLine($"Started transcription job: {transcriptionJobName}");
GetTranscriptionJobRequest request = new GetTranscriptionJobRequest {
TranscriptionJobName = transcriptionJobName
};
GetTranscriptionJobResponse response = null;
do
{
AppendStatusLine($"... {DateTime.Now} Waiting for transcription job to complete");
await Task.Delay(TimeSpan.FromSeconds(2));
response = await transcribeClient.GetTranscriptionJobAsync(request);
} while (response.TranscriptionJob.TranscriptionJobStatus == TranscriptionJobStatus.IN_PROGRESS);
if (response.TranscriptionJob.TranscriptionJobStatus == TranscriptionJobStatus.FAILED)
{
AppendStatusLine($"Transcription job failed: {response.TranscriptionJob.FailureReason}");
return;
}
AppendStatusLine("Job Done");
var transcriptionDocument = await httpClient.GetStringAsync(response.TranscriptionJob.Transcript.TranscriptFileUri);
var root = JsonConvert.DeserializeObject(transcriptionDocument) as JObject;
var sb = new StringBuilder();
foreach (JObject transcriptionNode in root["results"]["transcripts"])
{
if (sb.Length != 0)
{
sb.AppendLine("\n\n");
}
sb.Append(transcriptionNode["transcript"]);
}
this._ctlTranscription.Text = sb.ToString();
}
}
catch (Exception ex)
{
AppendStatusLine($"Unknown error: {ex.Message}");
}
finally
{
this._ctlBeginTranscription.IsEnabled = true;
}
}
public override async Task ProcessMessageAsync(Message message, ILambdaContext context)
{
if (message.Iterations < 0)
{
return;
}
var suffix = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
// Initiate describe voices request.
var describeVoiceResponse = await _pollyClient.DescribeVoicesAsync(new DescribeVoicesRequest {
// LanguageCode = "en-US"
});
// LogInfo(JsonConvert.SerializeObject(describeVoiceResponse.Voices));
var randomIndex = _rand.Next(describeVoiceResponse.Voices.Count);
var randomVoice = describeVoiceResponse.Voices[randomIndex];
LogInfo($"Selected random voice '{randomVoice.Name}' in {randomVoice.LanguageName}");
// Initiate speech synthesis request.
var synthesizeResponse = await _pollyClient.SynthesizeSpeechAsync(new SynthesizeSpeechRequest {
VoiceId = randomVoice.Id,
OutputFormat = OutputFormat.Mp3,
Text = message.Text
});
var audioStream = new MemoryStream();
await synthesizeResponse.AudioStream.CopyToAsync(audioStream);
audioStream.Position = 0;
// Ensure audio file is in an s3 bucket for Transcribe to consume.
await _s3Client.PutObjectAsync(new PutObjectRequest {
BucketName = _audioBucket,
Key = $"polly_{suffix}.mp3",
InputStream = audioStream
});
// Initiate Transcription Job
var transcriptionName = $"transcribe_{suffix}";
var mediaUri = $"https://s3-us-east-1.amazonaws.com/{_audioBucket}/polly_{suffix}.mp3";
var transcriptionResponse = await _transcribeClient.StartTranscriptionJobAsync(new StartTranscriptionJobRequest {
LanguageCode = "en-US",
MediaFormat = MediaFormat.Mp3,
Media = new Media {
MediaFileUri = mediaUri
},
OutputBucketName = _textBucket,
TranscriptionJobName = transcriptionName
});
// Wait for transcription job to complete
while (true)
{
await Task.Delay(TimeSpan.FromSeconds(3));
var transcriptionJobStatusReponse = await _transcribeClient.GetTranscriptionJobAsync(new GetTranscriptionJobRequest {
TranscriptionJobName = transcriptionName
});
if (transcriptionJobStatusReponse.TranscriptionJob.TranscriptionJobStatus == TranscriptionJobStatus.FAILED)
{
LogWarn("Transcribe job failed");
return;
}
if (transcriptionJobStatusReponse.TranscriptionJob.TranscriptionJobStatus == TranscriptionJobStatus.COMPLETED)
{
LogInfo("Job Completed!!");
break;
}
LogInfo("Checking job status... again.");
}
// Retrieve text file from s3 and extract the text
var s3Response = await _s3Client.GetObjectAsync(new GetObjectRequest {
BucketName = _textBucket,
Key = $"{transcriptionName}.json"
});
var text = await new StreamReader(s3Response.ResponseStream).ReadToEndAsync();
LogInfo(text);
var json = JObject.Parse(text);
var transcription = json["results"]["transcripts"][0]["transcript"].ToString();
LogInfo(transcription);
if (message.Iterations >= 0 && transcription.Length > 0)
{
await _snsClient.PublishAsync(new PublishRequest {
TopicArn = _topic,
Message = JsonConvert.SerializeObject(new Message {
Iterations = message.Iterations - 1,
Text = transcription,
Link = mediaUri
})
});
}
// :)
}