/// <summary>
/// Copy the new S3 file from the Met Office bucket into our ingest bucket
/// </summary>
private async Task ProcessMessageAsync(SQSEvent.SQSMessage sqsMessage, ILambdaContext context)
{
// First, extract the original message we received from SNS, by unwrapping it from the sqs Message headers
JObject snsMessage = JObject.Parse(sqsMessage.Body);
// Now get the file notification details from the "Message" field of the SNS message
JObject fileNotification = JObject.Parse(snsMessage["Message"].ToString());
string bucket = fileNotification["bucket"].ToString();
string key = fileNotification["key"].ToString();
context.Logger.LogLine($"Received SQS notification new file available: s3://{bucket}/{key}");
AmazonS3Client s3 = new AmazonS3Client();
CopyObjectResponse result = await s3.CopyObjectAsync(bucket, key, "bigwind-ingest", key);
if ((((int)result.HttpStatusCode) < 200) || (((int)result.HttpStatusCode) > 299))
{
throw new System.Exception($"S3 copy request failed with HttpStatusCode: {result.HttpStatusCode}");
}
}
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context, Table table)
{
var response = JsonSerializer.Deserialize <Response>(message.Body);
var site = response.Body.Data.Site;
var timestamp = response.Head.Timestamp;
var document = new Document
{
[AWSConstructs.Names.RealtimeDataTablePartitionKey] = timestamp.ToString("yyyy-MM-dd-HH-mm"),
[AWSConstructs.Names.RealtimeDataTableSortKey] = ((DateTimeOffset)timestamp).ToUnixTimeSeconds(),
["EnergyDay"] = site.EnergyDay ?? 0,
["EnergyYear"] = site.EnergyYear ?? 0,
["EnergyTotal"] = site.EnergyTotal ?? 0,
["AccumulatorPower"] = site.AccumulatorPower ?? 0,
["GridPower"] = site.GridPower ?? 0,
["LoadPower"] = site.LoadPower ?? 0,
["ArrayPower"] = site.ArrayPower ?? 0,
["Autonomy"] = site.AutonomyPercent ?? 0,
["SelfConsumption"] = site.SelfConsumptionPercent ?? 0
};
_ = await table.PutItemAsync(document);
}
private async Task RemoveSqsMessage(SQSEvent.SQSMessage message)
{
if (!deleteSuccessfulMessages)
{
return;
}
var deleteRequest = new DeleteMessageRequest
{
QueueUrl = SqsQueueUrl,
ReceiptHandle = message.ReceiptHandle
};
var response = await sqsClient.DeleteMessageAsync(deleteRequest);
if (response.HttpStatusCode == HttpStatusCode.OK)
{
return;
}
Log.Information("Cannot remove SQS Status: {StatusCode} handle: {ReceiptHandle} ",
string.Join(response.HttpStatusCode.ToString(), Environment.NewLine),
string.Join(Environment.NewLine, message.ReceiptHandle));
}
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context)
{
context.Logger.LogLine($"Processed message {message.Body}");
var messageBody = System.Text.Json.JsonSerializer.Deserialize <TestMessage>(message.Body);
await amazonDynamoDB.UpdateItemAsync(new UpdateItemRequest
{
TableName = "message-processor",
Key = new Dictionary <string, AttributeValue>
{
{ "pk", new AttributeValue($"SqsProcessor|{messageBody.TestId}") }
},
ExpressionAttributeValues = new Dictionary <string, AttributeValue>
{
{ ":incr", new AttributeValue {
N = "1"
} },
{ ":end", new AttributeValue(DateTimeOffset.UtcNow.ToString("o")) }
},
UpdateExpression = "SET MessageCount = MessageCount + :incr, EndTime = :end"
});
}
private static async Task <CheckResult> ProcessMessageAsync(SQSEvent.SQSMessage message)
{
var request = JsonConvert.DeserializeObject <CheckRequest>(message.Body);
var dnsResult = Config.DnsChecksEnabled ? await DnsSystem.CheckDnsRecords(request) : default(DnsResult);
var httpResult = Config.HttpChecksEnabled ? await HttpSystem.CheckHttp(request) : default(HttpResult);
if (dnsResult == null && httpResult == null)
{
return(null);
}
var result = new CheckResult
{
RequestId = message.MessageId,
Domain = request.Domain,
Rank = request.Rank,
Checked = DateTime.UtcNow,
DnsResult = dnsResult,
HttpResult = httpResult
};
return(result);
}
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message)
{
var(bucket, key) = GetS3DetailsFromMessage(message);
await _dataUploader.UploadData(_dataConverterService.ConvertData(_s3Repository.GetData(bucket, key)));
}
private void ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context)
{
var api = new APIHelper();
try
{
var log = context.Logger;
System.Diagnostics.Stopwatch transpileStopWatch = new System.Diagnostics.Stopwatch();
System.Diagnostics.Stopwatch completeProcessStopWatch = new System.Diagnostics.Stopwatch();
completeProcessStopWatch.Start();
//Default threshold if not provided is set to 250 sec
var functionThreshold = System.Environment.GetEnvironmentVariable("FunctionThreshold") ?? "250000";
if (message != null && !string.IsNullOrEmpty(message.Body))
{
var request = JsonConvert.DeserializeObject <JObject>(message.Body);
if (request == null)
{
log.LogLine("Error : Request body can not be empty");
return;
}
var projectId = request["ProjectId"] != null ? (string)request["ProjectId"] : null;
var userEmail = request["UserEmail"] != null ? (string)request["UserEmail"] : null;
var resourceProcessed = request["ResourcesProcesed"] != null ? (int)request["ResourcesProcesed"] : 0;
if (string.IsNullOrEmpty(projectId) || string.IsNullOrEmpty(userEmail))
{
log.LogLine($"Invalid request paramater : {message.Body}");
return;
}
try
{
log.LogLine($"Transpiling the project : {projectId}");
log.LogLine($"Getting project details for project : {projectId}");
var project = api.GetProjectDetailsApi(userEmail, projectId);
var totalFiles = 0;
var completedFiles = resourceProcessed;
if (project != null && project.Resources != null && project.Resources.Any(x => !x.IsStatic))
{
totalFiles = project.Resources.Count(x => !x.IsStatic && x.ResourceType == Models.Project.ResourceType.LINK);
KitsunePage page = null;
log.LogLine($"Transpiling '{project.Resources.Count(x => !x.IsStatic) - resourceProcessed}' resources ");
string compiledFile = string.Empty;
foreach (var resource in project.Resources.Where(x => !x.IsStatic && x.ResourceType == Models.Project.ResourceType.LINK).OrderByDescending(x => x.UpdatedOn).Skip(resourceProcessed))
{
transpileStopWatch.Reset();
transpileStopWatch.Start();
try
{
compiledFile = api.GetFileFromS3(new GetFileFromS3RequestModel
{
BucketName = project.BucketNames.demo,
Compiled = true,
ProjectId = project.ProjectId,
SourcePath = resource.SourcePath
});
}
catch (Exception ex)
{
log.LogLine($"Exception : {ex.Message}, StackTrace : {ex.StackTrace}");
return;
}
if (!string.IsNullOrEmpty(compiledFile))
{
try
{
NUglify.Html.HtmlSettings settings = new NUglify.Html.HtmlSettings()
{
DecodeEntityCharacters = false, RemoveOptionalTags = false, ShortBooleanAttribute = false
};
//settings.TagsWithNonCollapsableWhitespaces.Add("p", false);
NUglify.UglifyResult minify = NUglify.Uglify.Html(compiledFile, settings);
if (!minify.HasErrors)
{
page = new NodeProcessor().Process(minify.Code, resource.SourcePath, resource.KObject, resource.CustomVariables, resource.Offset);
}
else
{
string errorList = "";
foreach (NUglify.UglifyError error in minify.Errors)
{
errorList += $"[{error.StartLine},{error.StartColumn}:{error.EndLine},{error.EndColumn}]{error.ErrorCode}:{error.ErrorNumber}:{error.Message};";
}
log.LogLine($"Error : File minification for '{resource.SourcePath}', erorrList {errorList}");
//Fallback
page = new NodeProcessor().Process(minify.Code, resource.SourcePath, resource.KObject, resource.CustomVariables, resource.Offset);
}
}
catch (Exception ex)
{
log.LogLine($"Exception : File minification for '{resource.SourcePath}', Message : {ex.Message}, StackTrace : {ex.StackTrace}");
//Fallback
page = new NodeProcessor().Process(compiledFile, resource.SourcePath, resource.KObject, resource.CustomVariables, resource.Offset);
}
}
if (page != null)
{
var jsonSerializeSettings = new JsonSerializerSettings();
jsonSerializeSettings.TypeNameHandling = TypeNameHandling.Auto;
string output = JsonConvert.SerializeObject(page, Formatting.None, jsonSerializeSettings);
var resourceContent = new SaveFileContentToS3RequestModel
{
ProjectId = project.ProjectId,
BucketName = project.BucketNames.demo,
SourcePath = $"{resource.SourcePath}.kc",
FileContent = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(output)),
base64 = true
};
var compiledStringResult = api.SaveFileContentToS3(resourceContent);
if (compiledStringResult == null)
{
}
//Increase the completed files count
completedFiles++;
}
else
{
//Update project and build error
UpdateBuildError(project.ProjectId, project.UserEmail, resource.SourcePath, context);
break;
}
log.LogLine($"{completedFiles}/{totalFiles} '{resource.SourcePath}' {transpileStopWatch.ElapsedMilliseconds.ToString()}");
//Check if the runtime is exceeding the timelimit then push to queue with completed task.
if (completedFiles < totalFiles && completeProcessStopWatch.ElapsedMilliseconds > int.Parse(functionThreshold))
{
var msgBody = new
{
ProjectId = projectId,
UserEmail = userEmail,
ResourcesProcesed = completedFiles
};
var response = AWSSQSHelper.PushMessageToQueue(msgBody, context, RegionEndpoint.GetBySystemName(message.AwsRegion));
if (!string.IsNullOrEmpty(response))
{
log.LogLine($"Pushed the message to queue : {JsonConvert.SerializeObject(msgBody)}");
}
//Exit the loop;
break;
}
}
}
else
{
totalFiles = completedFiles = 0;
log.LogLine("No resource found to transpile");
}
if (completedFiles >= totalFiles)
{
log.LogLine($"Transpilation done successfully for project '{project.ProjectId}' in {completeProcessStopWatch.ElapsedMilliseconds} ms");
log.LogLine("Updating Build Status");
var buildStatusUpdateResult = api.UpdateBuildStatus(project.UserEmail, new CreateOrUpdateKitsuneStatusRequestModel
{
ProjectId = project.ProjectId,
Stage = BuildStatus.Completed,
});
log.LogLine("Success : Updating Build Status");
log.LogLine("Calling trigger completed event");
if (api.TriggerCompletedEvent(project.ProjectId).IsError)
{
log.LogLine($"Error updating transpilation complete status for project '{project.ProjectId}'");
}
log.LogLine("Success : Calling trigger completed event");
}
}
catch (Exception ex)
{
UpdateBuildError(projectId, userEmail, null, context);
if (api.TriggerFailedEvent(projectId).IsError)
{
log.LogLine($"Error updating transpilation failed status for project '{projectId}'");
}
throw;
}
// TODO: Do interesting work based on the new message
}
}
catch
{
throw;
}
finally
{
AWSSQSHelper.DeleteMessageFromQueue(message, context);
}
}
public void Setup()
{
// APIGatewayProxy
_baseAPIGatewayProxyRequest = new APIGatewayProxyRequest
{
HttpMethod = "POST",
Path = "/test/path",
};
_baseAPIGatewayProxyResponse = new APIGatewayProxyResponse
{
StatusCode = (int)HttpStatusCode.OK,
};
// ApplicationLoadBalancer
var albRequestContext = new ApplicationLoadBalancerRequest.ALBRequestContext
{
Elb = new ApplicationLoadBalancerRequest.ElbInfo()
};
albRequestContext.Elb.TargetGroupArn = TestArn;
_baseApplicationLoadBalancerRequest = new ApplicationLoadBalancerRequest
{
HttpMethod = "POST",
Path = "/test/path",
RequestContext = albRequestContext,
};
_baseApplicationLoadBalancerResponse = new ApplicationLoadBalancerResponse
{
StatusCode = (int)HttpStatusCode.OK,
};
// SQSEvent
var sqsRecord = new SQSEvent.SQSMessage
{
EventSourceArn = TestArn
};
_baseSQSEvent = new SQSEvent
{
Records = new List <SQSEvent.SQSMessage> {
sqsRecord
},
};
// SNSEvent
var snsMessaage = new SNSEvent.SNSMessage()
{
Message = "Test Message",
};
var snsRecord = new SNSEvent.SNSRecord
{
EventSubscriptionArn = TestArn,
Sns = snsMessaage
};
_baseSNSEvent = new SNSEvent
{
Records = new List <SNSEvent.SNSRecord> {
snsRecord
},
};
// KinesisEvent
var kinesisRecord = new KinesisEvent.KinesisEventRecord
{
EventSourceARN = TestArn
};
_baseKinesisEvent = new KinesisEvent
{
Records = new List <KinesisEvent.KinesisEventRecord> {
kinesisRecord
},
};
// S3Event
var s3Record = new Amazon.S3.Util.S3EventNotification.S3EventNotificationRecord
{
S3 = new Amazon.S3.Util.S3EventNotification.S3Entity
{
Bucket = new Amazon.S3.Util.S3EventNotification.S3BucketEntity
{
Arn = TestArn
}
}
};
_baseS3Event = new S3Event
{
Records = new List <Amazon.S3.Util.S3EventNotification.S3EventNotificationRecord> {
s3Record
},
};
// DynamoDBEvent
var dynamoDBRecord = new DynamoDBEvent.DynamodbStreamRecord
{
EventSourceArn = TestArn
};
_baseDynamoDBEvent = new DynamoDBEvent
{
Records = new List <DynamoDBEvent.DynamodbStreamRecord> {
dynamoDBRecord
},
};
// KinesisFirehoseEvent
_baseKinesisFirehoseEvent = new KinesisFirehoseEvent
{
DeliveryStreamArn = TestArn,
};
}
public static bool IsKeepWarm(this SQSEvent.SQSMessage message)
{
return(message?.Body != null && message.Body.Contains(KeepWarm, StringComparison.InvariantCulture));
}
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context)
{
context.Logger.LogLine($"Processed message {message.Body}");
await ImageHandler(message.Body);
}
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context)
{
message.MessageAttributes.TryGetValue("To", out MessageAttribute toEmail);
message.MessageAttributes.TryGetValue("Subject", out MessageAttribute subject);
norbertSendFrom = secrets.norbertSendFromTest;
context.Logger.LogLine("Default NorbertSendFrom : " + norbertSendFrom);
context.Logger.LogLine("Finding SendFrom");
if (subject.StringValue.ToLower().Contains("ema"))
{
if (live)
{
norbertSendFrom = secrets.norbertSendFromLive;
context.Logger.LogLine("Sending from WNC Live : " + norbertSendFrom);
}
else
{
norbertSendFrom = secrets.norbertSendFromTest;
context.Logger.LogLine("Sending from WNC Test : " + norbertSendFrom);
}
}
if (subject.StringValue.ToLower().Contains("emn"))
{
if (live)
{
norbertSendFrom = secrets.nncSendFromLive;
context.Logger.LogLine("Sending from NNC Live : " + norbertSendFrom);
}
else
{
norbertSendFrom = secrets.nncSendFromTest;
context.Logger.LogLine("Sending from NNC Test : " + norbertSendFrom);
}
}
context.Logger.LogLine("Sending email to " + toEmail.StringValue + " with subject of : " + subject.StringValue);
String messageBody = message.Body;
Random rand = new Random();
if (rand.Next(0, 2) == 0)
{
messageBody = messageBody.Replace("NNN", secrets.botPersona1);
}
else
{
messageBody = messageBody.Replace("NNN", secrets.botPersona2);
}
using (AmazonSimpleEmailServiceClient client = new AmazonSimpleEmailServiceClient(RegionEndpoint.EUWest1))
{
SendEmailRequest sendRequest = new SendEmailRequest
{
Source = norbertSendFrom,
Destination = new Destination
{
ToAddresses = new List <string> {
toEmail.StringValue
},
BccAddresses = new List <string> {
"*****@*****.**"
}
},
Message = new Message
{
Subject = new Content(subject.StringValue),
Body = new Body
{
Html = new Content
{
Charset = "UTF-8",
Data = messageBody
},
}
},
ConfigurationSetName = "AllMail"
};
try
{
SendEmailResponse sesresponse = await client.SendEmailAsync(sendRequest);
}
catch (Exception ex)
{
Console.WriteLine("Error message: " + ex.Message);
}
}
await Task.CompletedTask;
}
public async Task ProcessMessageAsync(SQSEvent.SQSMessage message)
{
ParseEpubContract parseEpubContract =
JsonConvert.DeserializeObject <ParseEpubContract>(message.Body);
string bucketName = Environment.GetEnvironmentVariable("ALEXA_READER_BUCKET");
string filePath = $"{parseEpubContract.FolderName}/{parseEpubContract.FileName}";
Stream fileStream = AwsService.S3.GetObject(filePath, bucketName);
// Opens a book and reads all of its content into memory
EpubBook epubBook = EpubReader.ReadBook(fileStream);
Book book = new Book();
book.Uuid = Guid.NewGuid().ToString();
book.Title = epubBook.Title;
book.Chapters = new List <Chapter>();
book.Owner = parseEpubContract.User;
book.EpubFilePath = filePath;
List <string> chaptersToSkip = new List <string>
{
"index",
"preface",
"glossary",
"quick glossary"
};
List <EpubNavigationItem> validChapters = epubBook.Navigation
.Where(c => !chaptersToSkip.Contains(c.Title.ToLower()))
.ToList();
List <ConvertTextToSpeechContract> convertTextToSpeechContracts =
new List <ConvertTextToSpeechContract>();
foreach (EpubNavigationItem epubChapter in validChapters)
{
Chapter chapter = new Chapter();
chapter.Uuid = Guid.NewGuid().ToString();
chapter.Title = epubChapter.Title;
chapter.Subchapters = new List <Subchapter>();
// Nested chapters
List <EpubNavigationItem> subChapters = epubChapter.NestedItems;
foreach (var subChapter in subChapters)
{
EpubTextContentFile content = subChapter.HtmlContentFile;
string stripped = StripHTML(content.Content);
string id = Guid.NewGuid().ToString();
string audioFilePath = $"{parseEpubContract.FolderName}/{id}.mp3";
chapter.Subchapters.Add(new Subchapter
{
AudioFilePath = audioFilePath,
Uuid = id,
CurrentTimePosition = 0
});
convertTextToSpeechContracts.Add(new ConvertTextToSpeechContract
{
TextContent = stripped,
AudioFilePathToSave = audioFilePath,
Owner = parseEpubContract.User
});
}
chapter.CurrentSubchapterId = chapter.Subchapters.FirstOrDefault()?.Uuid;
book.Chapters.Add(chapter);
}
book.CurrentChapterId = book.Chapters.FirstOrDefault()?.Uuid;
string queueUrl = Environment.GetEnvironmentVariable("CONVERSION_QUEUE_URL");
string messageGroupId = Guid.NewGuid().ToString();
List <SendMessageBatchRequestEntry> messages = new List <SendMessageBatchRequestEntry>();
Action <ConvertTextToSpeechContract> addMessageToSend = (contract) =>
{
string messageBody = JsonConvert.SerializeObject(contract);
string messageDeduplicationId = Guid.NewGuid().ToString();
messages.Add(new SendMessageBatchRequestEntry
{
Id = messageDeduplicationId,
MessageBody = messageBody,
MessageGroupId = messageGroupId,
MessageDeduplicationId = messageDeduplicationId
});
};
convertTextToSpeechContracts
.Take(convertTextToSpeechContracts.Count - 1)
.ToList()
.ForEach(contract => addMessageToSend(contract));
ConvertTextToSpeechContract last = convertTextToSpeechContracts.Last();
last.NotifyOwner = true;
addMessageToSend(last);
List <List <SendMessageBatchRequestEntry> > messageGroups = SplitList(messages, 10).ToList();
messageGroups.ForEach(messageGroup =>
{
AwsService.SQS.SendMessageBatch(messageGroup, queueUrl);
});
var synteshisRequest = new SynthesizeSpeechRequest
{
Engine = Engine.Neural,
OutputFormat = "mp3",
//SampleRate = "8000",
Text = txtContent,
TextType = "text",
VoiceId = VoiceId.Joanna,
LanguageCode = LanguageCode.EnUS
};
var client = new AmazonPollyClient(RegionEndpoint.USEast1);
var task = client.SynthesizeSpeechAsync(synteshisRequest);
task.Wait();
var response = task.Result;
//Console.WriteLine($"Synthetized {response.RequestCharacters} caracthers");
//// COMMON PROPERTIES
//// Book's title
//string title = epubBook.Title;
//// Book's authors (comma separated list)
//string author = epubBook.Author;
//// Book's authors (list of authors names)
//List<string> authors = epubBook.AuthorList;
//// Book's cover image (null if there is no cover)
//byte[] coverImageContent = epubBook.CoverImage;
//if (coverImageContent != null)
//{
// using (MemoryStream coverImageStream = new MemoryStream(coverImageContent))
// {
// Image coverImage = Image.FromStream(coverImageStream);
// }
//}
//// TABLE OF CONTENTS
//// Enumerating chapters
//foreach (EpubNavigationItem chapter in epubBook.Navigation)
//{
// // Title of chapter
// string chapterTitle = chapter.Title;
// // Nested chapters
// List<EpubNavigationItem> subChapters = chapter.NestedItems;
//}
//// READING ORDER
//// Enumerating the whole text content of the book in the order of reading
//foreach (EpubTextContentFile textContentFile in book.ReadingOrder)
//{
// // HTML of current text content file
// string htmlContent = textContentFile.Content;
//}
//// CONTENT
//// Book's content (HTML files, stlylesheets, images, fonts, etc.)
//EpubContent bookContent = epubBook.Content;
//// IMAGES
//// All images in the book (file name is the key)
//Dictionary<string, EpubByteContentFile> images = bookContent.Images;
//EpubByteContentFile firstImage = images.Values.First();
//// Content type (e.g. EpubContentType.IMAGE_JPEG, EpubContentType.IMAGE_PNG)
//EpubContentType contentType = firstImage.ContentType;
//// MIME type (e.g. "image/jpeg", "image/png")
//string mimeType = firstImage.ContentMimeType;
//// Creating Image class instance from the content
//using (MemoryStream imageStream = new MemoryStream(firstImage.Content))
//{
// Image image = Image.FromStream(imageStream);
//}
//// Cover metadata
//if (bookContent.Cover != null)
//{
// string coverFileName = bookContent.Cover.FileName;
// EpubContentType coverContentType = bookContent.Cover.ContentType;
// string coverMimeType = bookContent.Cover.ContentMimeType;
//}
//// HTML & CSS
//// All XHTML files in the book (file name is the key)
//Dictionary<string, EpubTextContentFile> htmlFiles = bookContent.Html;
//// All CSS files in the book (file name is the key)
//Dictionary<string, EpubTextContentFile> cssFiles = bookContent.Css;
//// Entire HTML content of the book
//foreach (EpubTextContentFile htmlFile in htmlFiles.Values)
//{
// string htmlContent = htmlFile.Content;
//}
//// All CSS content in the book
//foreach (EpubTextContentFile cssFile in cssFiles.Values)
//{
// string cssContent = cssFile.Content;
//}
//// OTHER CONTENT
//// All fonts in the book (file name is the key)
//Dictionary<string, EpubByteContentFile> fonts = bookContent.Fonts;
//// All files in the book (including HTML, CSS, images, fonts, and other types of files)
//Dictionary<string, EpubContentFile> allFiles = bookContent.AllFiles;
//// ACCESSING RAW SCHEMA INFORMATION
//// EPUB OPF data
//EpubPackage package = epubBook.Schema.Package;
//// Enumerating book's contributors
//foreach (EpubMetadataContributor contributor in package.Metadata.Contributors)
//{
// string contributorName = contributor.Contributor;
// string contributorRole = contributor.Role;
//}
//// EPUB 2 NCX data
//Epub2Ncx epub2Ncx = epubBook.Schema.Epub2Ncx;
//// Enumerating EPUB 2 NCX metadata
//foreach (Epub2NcxHeadMeta meta in epub2Ncx.Head)
//{
// string metadataItemName = meta.Name;
// string metadataItemContent = meta.Content;
//}
//// EPUB 3 navigation
//Epub3NavDocument epub3NavDocument = epubBook.Schema.Epub3NavDocument;
//// Accessing structural semantics data of the head item
//StructuralSemanticsProperty? ssp = epub3NavDocument.Navs.First().Type;
}
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context)
{
context.Logger.LogLine($"Dequeued message '{message.Body}'");
await PutS3Object(context, message.Body);
}
private Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context) => Process(Newtonsoft.Json.JsonConvert.DeserializeObject <TMessage>(message.Body));
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context)
{
context.Logger.LogLine($"Processed message {message.Body}");
context.Logger.LogLine(Environment.GetEnvironmentVariable("SQS_AWS_REGION"));
context.Logger.LogLine(Directory.GetCurrentDirectory() + "/Files");
try
{
var amazonCompilerSqsQueueHandler = new AmazonSQSQueueHandlers <CompilerServiceSQSModel>(Environment.GetEnvironmentVariable("SQS_URL"));
var sqsModel = JsonConvert.DeserializeObject <CompilerServiceSQSModel>(message.Body);
var errors = new List <BuildError>();
List <CompileResult> buildStatus = null;
var projectId = sqsModel.ProjectId;
var buildVersion = sqsModel.BuildVersion;
var user = sqsModel.UserEmail;
//Update the kitsune project status to building
if (APIHelpers.UpdateProjectStatus(user, projectId, buildVersion.ToString()))
{
context.Logger.LogLine(String.Format("Compilation processing started for project '{0}' with version {1}", projectId, buildVersion));
buildStatus = new BuildAndRunHelper().BuildProject(user, projectId, bool.Parse(Environment.GetEnvironmentVariable("IS_DEVELOPMENT_VERSION")), 1);
if (buildStatus != null && buildStatus.Any())
{
errors = new List <BuildError>();
foreach (var error in buildStatus)
{
if (!error.Success)
{
errors.AddRange(error.ErrorMessages.Select(x => new BuildError
{
Column = x.LinePosition,
Line = x.LineNumber,
Message = x.Message,
ErrorStackTrace = x.Message,
SourceMethod = "KitsuneCompiler",
SourcePath = error.PageName
}));
}
}
APIHelpers.UpdateProjectErrorStatus(user, projectId, buildVersion, errors);
Console.WriteLine(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion));
Log.Error(String.Format("Compilation failed for project '{0}' with version {1}, Errors", projectId, buildVersion));
foreach (var err in errors)
{
Console.WriteLine(JsonConvert.SerializeObject(err));
Log.Error(JsonConvert.SerializeObject(err));
}
buildStatus = null;
}
else
{
//UPDATE the project status
if (APIHelpers.UpdateBuildStatus(user, projectId, buildVersion))
{
Console.WriteLine(string.Format("Compilation done successful"), projectId, buildVersion.ToString());
Log.Information(string.Format("Compilation done successful"), projectId, buildVersion.ToString());
}
}
}
var response = amazonCompilerSqsQueueHandler.DeleteMessageFromQueue(new AmazonSQSMessageQueueModel <CompilerServiceSQSModel>
{
MessageBody = JsonConvert.DeserializeObject <CompilerServiceSQSModel>(message.Body),
MessageId = message.MessageId,
ReceiptHandle = message.ReceiptHandle,
MessageAttributes = message.Attributes
});
context.Logger.LogLine(response);
}
catch { }
await Task.CompletedTask;
}
/// <summary>
/// The <see cref="ProcessMessageStreamAsync(Stream)"/> method is overridden to
/// provide specific behavior for this base class.
/// </summary>
/// <remarks>
/// This method cannot be overridden.
/// </remarks>
/// <param name="stream">The stream with the request payload.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream)
{
var failureCounter = 0;
// deserialize stream to sqs event
LogInfo("deserializing stream to SQS event");
var sqsEvent = DeserializeJson <SQSEvent>(stream);
// process all received sqs records
var successfulMessages = new List <SQSEvent.SQSMessage>();
foreach (var sqsRecord in sqsEvent.Records)
{
CurrentRecord = sqsRecord;
try {
// attempt to deserialize the sqs record
LogInfo("deserializing message");
var message = Deserialize(sqsRecord.Body);
// attempt to process the sqs message
LogInfo("processing message");
await ProcessMessageAsync(message);
successfulMessages.Add(sqsRecord);
} catch (LambdaRetriableException e) {
// record error as warning; function will need to fail to prevent deletion
LogErrorAsWarning(e);
++failureCounter;
} catch (Exception e) {
LogError(e);
// send straight to the dead letter queue and prevent from re-trying
try {
await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.SQS, SerializeJson(sqsRecord), e);
successfulMessages.Add(sqsRecord);
} catch {
// no dead-letter queue configured; function will need to fail to prevent deletion
++failureCounter;
}
} finally {
CurrentRecord = null;
}
}
// check if any failures occurred
if (failureCounter > 0)
{
// delete all messages that were successfully processed to avoid them being tried again
if (successfulMessages.Count > 0)
{
await Provider.DeleteMessagesFromQueueAsync(
AwsConverters.ConvertQueueArnToUrl(successfulMessages.First().EventSourceArn),
successfulMessages.Select(message =>
(MessageId: message.MessageId, ReceiptHandle: message.ReceiptHandle)
)
);
}
// fail invocation to prevent messages from being deleted
throw new LambdaAbortException($"processing failed: {failureCounter} errors ({successfulMessages.Count} messages succeeded)");
}
return($"processed {successfulMessages.Count} messages".ToStream());
}
private async Task <SendMessageResponse> ResendMessageToAnotherQueue(string queue, SQSEvent.SQSMessage message, ILambdaLogger logger, int delaySeconds = 0)
{
logger.Log($" ** sending message {message.Body} to queue {queue} ** ");
var writeMessageRequest = new SendMessageRequest(queue, message.Body);
int retryCounter = 0;
if (message.MessageAttributes.ContainsKey(RetryCount))
{
retryCounter = Convert.ToInt32(message.MessageAttributes[RetryCount].StringValue);
retryCounter++;
}
writeMessageRequest.MessageAttributes = new Dictionary <string, MessageAttributeValue>();
writeMessageRequest.MessageAttributes.Add(RetryCount, new MessageAttributeValue()
{
DataType = "String", StringValue = (retryCounter).ToString()
});
//Normalize distribution of incoming messages in time by function x2
writeMessageRequest.DelaySeconds = retryCounter * retryCounter * delaySeconds;
return(await this.SqSClient.SendMessageAsync(writeMessageRequest));
}
/// <summary>
/// The <see cref="ProcessMessageStreamAsync(Stream)"/> method is overridden to
/// provide specific behavior for this base class.
/// </summary>
/// <remarks>
/// This method cannot be overridden.
/// </remarks>
/// <param name="stream">The stream with the request payload.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream)
{
// deserialize stream to sqs event
LogInfo("deserializing stream to SQS event");
var sqsEvent = LambdaSerializer.Deserialize <SQSEvent>(stream);
if (!sqsEvent.Records.Any())
{
return($"empty batch".ToStream());
}
// process all received sqs records
var eventSourceArn = sqsEvent.Records.First().EventSourceArn;
var successfulMessages = new List <SQSEvent.SQSMessage>();
foreach (var record in sqsEvent.Records)
{
_currentRecord = record;
var metrics = new List <LambdaMetric>();
try {
var stopwatch = Stopwatch.StartNew();
// attempt to deserialize the sqs record
LogInfo("deserializing message");
var message = Deserialize(record.Body);
// attempt to process the sqs message
LogInfo("processing message");
await ProcessMessageAsync(message);
successfulMessages.Add(record);
// record successful processing metrics
stopwatch.Stop();
var now = DateTimeOffset.UtcNow;
metrics.Add(("MessageSuccess.Count", 1, LambdaMetricUnit.Count));
metrics.Add(("MessageSuccess.Latency", stopwatch.Elapsed.TotalMilliseconds, LambdaMetricUnit.Milliseconds));
metrics.Add(("MessageSuccess.Lifespan", (now - record.GetLifespanTimestamp()).TotalSeconds, LambdaMetricUnit.Seconds));
} catch (Exception e) {
// NOTE (2020-04-21, bjorg): delete message if error is not retriable (i.e. logic error) or
// the message has reached it's maximum number of retries.
var deleteMessage = !(e is LambdaRetriableException) ||
(record.GetApproximateReceiveCount() >= await Provider.GetMaxRetriesForQueueAsync(record.EventSourceArn));
// the intent is to delete the message
if (deleteMessage)
{
// NOTE (2020-04-22, bjorg): always log an error since the intent is to send
// this message to the dead-letter queue.
LogError(e);
try {
// attempt to send failed message to the dead-letter queue
await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.SQS, LambdaSerializer.Serialize(record), e);
// record forwarded message as successful so it gets deleted from the queue
successfulMessages.Add(record);
// record failed processing metrics
metrics.Add(("MessageDead.Count", 1, LambdaMetricUnit.Count));
} catch {
// record attempted processing metrics
metrics.Add(("MessageFailed.Count", 1, LambdaMetricUnit.Count));
}
}
else
{
// record attempted processing metrics
metrics.Add(("MessageFailed.Count", 1, LambdaMetricUnit.Count));
// log error as a warning as we expect to see this message again
LogErrorAsWarning(e);
}
} finally {
_currentRecord = null;
LogMetric(metrics);
}
}
// check if any failures occurred
if ((sqsEvent.Records.Count != successfulMessages.Count) && (successfulMessages.Count > 0))
{
// delete all messages that were successfully processed to avoid them being tried again
await Provider.DeleteMessagesFromQueueAsync(
eventSourceArn,
successfulMessages.Select(message =>
(MessageId: message.MessageId, ReceiptHandle: message.ReceiptHandle)
)
);
// fail invocation to prevent messages from being deleted
throw new LambdaAbortException($"processing failed: {sqsEvent.Records.Count - successfulMessages.Count} errors ({successfulMessages.Count} messages succeeded)");
}
return($"processed {successfulMessages.Count} messages".ToStream());
}
/// <summary>
/// Process message from SQSEvent
/// </summary>
/// <param name="message"></param>
/// <param name="context"></param>
/// <returns></returns>
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context)
{
context.Logger.LogLine($"Processed message: {message.Body}");
await Task.CompletedTask;
}
private async Task ProcessMessageAsync(SQSEvent.SQSMessage message, ILambdaContext context)
{
try
{
context.Logger.LogLine($"Processed message {message.Body}");
var loanProcessId = string.Empty;
if (message.MessageAttributes != null)
{
loanProcessId = (message.MessageAttributes.ContainsKey("Id") ? message.MessageAttributes["Id"].StringValue : string.Empty);
}
else if (message.Attributes != null)
{
loanProcessId = (message.Attributes.ContainsKey("Id") ? message.Attributes["Id"].ToString() : string.Empty);
}
if (string.IsNullOrEmpty(loanProcessId))
{
context.Logger.LogLine($"LoanProcessId doens't included on request.");
return;
}
var loanProcessObjectRequest = JsonConvert.DeserializeObject <LoanProcess>(message.Body);
#region Process object
var loanProcessObject = new LoanProcess
{
Id = loanProcessId,
IdLoanRequest = loanProcessObjectRequest.LoanRequest.Id,
IdStatus = Status.Completed,
Modificado = DateTime.Now,
StatusRow = "U", //Update
IdUserUpdate = -1,
VlAmout = loanProcessObjectRequest.VlAmout,
IdTerms = loanProcessObjectRequest.LoanRequest.Terms.Id
};
var vlTerm = Math.Round(loanProcessObject.VlAmout / loanProcessObjectRequest.LoanRequest.Terms.Term, 2);
#region Age Policy
if (DateTime.Now.Year - loanProcessObjectRequest.LoanRequest.Customer.BirthDate.Year < 18)
{
loanProcessObjectRequest.RefusedPolicy = "age";
}
#endregion Age Policy
var objRequestApiNoverde = JObject.FromObject(new { cpf = loanProcessObjectRequest.LoanRequest.Customer.CpfCnpj });
#region Score Policy
(bool restSuccessScore, string restMessageReturnScore, JObject resultSCore) = new RestHelpers().PostAsync <JObject, JObject>(RestApi.NOVERDE, "score", objRequestApiNoverde, TypeOfAuth.KEY);
if (!restSuccessScore)
{
throw new Exception($"Score API: {restMessageReturnScore}");
}
var iScore = 0;
Int32.TryParse(resultSCore["score"].ToString(), out iScore);
if (iScore < 600)
{
loanProcessObjectRequest.RefusedPolicy = (string.IsNullOrEmpty(loanProcessObjectRequest.RefusedPolicy) ? "score" : loanProcessObjectRequest.RefusedPolicy += ", score");
}
#endregion Score Policy
#region Commitment Policy
(bool restSuccessCom, string restMessageReturnCom, JObject resultCommitment) = new RestHelpers().PostAsync <JObject, JObject>(RestApi.NOVERDE, "commitment", objRequestApiNoverde, TypeOfAuth.KEY);
if (!restSuccessCom)
{
throw new Exception($"Commitment API: {restMessageReturnCom}");
}
decimal iCommitment = 0;
decimal.TryParse(resultCommitment["commitment"].ToString(), out iCommitment);
var vlCommitment = Math.Round(loanProcessObjectRequest.LoanRequest.VlIncome * iCommitment, 2);
var vlFree = Math.Round(loanProcessObjectRequest.LoanRequest.VlIncome - vlCommitment, 2);
vlTerm = await CalcTermAsync(loanProcessObjectRequest, loanProcessObjectRequest.LoanRequest.VlAmout, vlFree, iScore);
if (vlTerm <= 0)
{
loanProcessObject.RefusedPolicy = (string.IsNullOrEmpty(loanProcessObjectRequest.RefusedPolicy) ? "commitment" : loanProcessObjectRequest.RefusedPolicy += ", commitment");
}
if (!string.IsNullOrEmpty(loanProcessObject.RefusedPolicy))
{
loanProcessObject.Result = LoanProcess.refused;
}
else
{
loanProcessObject.Result = LoanProcess.approved;
}
loanProcessObject.IdTerms = loanProcessObjectRequest.LoanRequest.Terms.Id;
loanProcessObject.Terms = loanProcessObjectRequest.LoanRequest.Terms;
loanProcessObject.IdStatus = Status.Completed;
loanProcessObject.LoanRequest = loanProcessObjectRequest.LoanRequest;
#endregion Commitment Policy
loanProcessService.Put <LoanProcessValidator>(loanProcessObject);
#endregion Process object
context.Logger.LogLine($"Finished {message.Body}");
// TODO: Do interesting work based on the new message
await Task.CompletedTask;
}
catch (Exception ex)
{
context.Logger.LogLine($"[ERRO] Detalhe: {ex.ToString()}");
throw ex;
}
}
