private async Task SendMessageToConnection(AnAction action, string connectionId)
{
var json = LambdaSerializer.Serialize <object>(action);
if (DebugLoggingEnabled)
{
LogDebug($"Post to connection: {connectionId}\n{{0}}", json);
}
else
{
LogInfo($"Post to connection: {connectionId}");
}
// attempt to send serialized message to connection
var messageBytes = Encoding.UTF8.GetBytes(json);
try {
await _amaClient.PostToConnectionAsync(new PostToConnectionRequest {
ConnectionId = connectionId,
Data = new MemoryStream(messageBytes)
});
} catch (AmazonServiceException e) when(e.StatusCode == System.Net.HttpStatusCode.Gone)
{
// HTTP Gone status code indicates the connection has been closed; nothing to do
} catch (Exception e) {
LogErrorAsWarning(e, "PostToConnectionAsync() failed on connection {0}", connectionId);
}
}
public override async Task <FunctionResponse> ProcessMessageAsync(FunctionRequest request)
{
LogInfo("Deserialized using {1}: {0}", LambdaSerializer.Serialize(request), LambdaSerializer.GetType().FullName);
return(new FunctionResponse {
Bar = request.Bar
});
}
public async Task SendMessageAsync(Message request)
{
// enumerate open connections
var connections = await _connections.GetAllRowsAsync();
LogInfo($"Found {connections.Count()} open connection(s)");
// attempt to send message on all open connections
var messageBytes = Encoding.UTF8.GetBytes(LambdaSerializer.Serialize(new Message {
From = request.From,
Text = request.Text
}));
var outcomes = await Task.WhenAll(connections.Select(async(connectionId, index) => {
LogInfo($"Post to connection {index}: {connectionId}");
try {
await _amaClient.PostToConnectionAsync(new PostToConnectionRequest {
ConnectionId = connectionId,
Data = new MemoryStream(messageBytes)
});
} catch (AmazonServiceException e) when(e.StatusCode == System.Net.HttpStatusCode.Gone)
{
LogInfo($"Deleting gone connection: {connectionId}");
await _connections.DeleteRowAsync(connectionId);
return(false);
} catch (Exception e) {
LogErrorAsWarning(e, "PostToConnectionAsync() failed");
return(false);
}
return(true);
}));
LogInfo($"Data sent to {outcomes.Count(result => result)} connections");
}
private CloudFormationResourceRequest <TProperties> DeserializeStream(Stream stream)
{
// read stream into memory
LogInfo("reading message stream");
string body;
using (var reader = new StreamReader(stream)) {
body = reader.ReadToEnd();
}
// deserialize stream into a generic JSON object
LogInfo("deserializing request");
var json = JsonConvert.DeserializeObject <JObject>(body);
// determine if the custom resource request is wrapped in an SNS message
// or if it is a direct invocation by the CloudFormation service
if (json.TryGetValue("Records", out _))
{
// deserialize SNS event
LogInfo("deserializing SNS event");
var snsEvent = json.ToObject <SNSEvent>();
// extract message from SNS event
LogInfo("deserializing message");
var messageBody = snsEvent.Records.First().Sns.Message;
// deserialize message into a cloudformation request
return(LambdaSerializer.Deserialize <CloudFormationResourceRequest <TProperties> >(messageBody));
}
else
{
// deserialize generic JSON into a cloudformation request
return(json.ToObject <CloudFormationResourceRequest <TProperties> >());
}
}
/// <summary>
/// The <see cref="ProcessMessageStreamAsync(Stream)"/> method is overridden to
/// provide specific behavior for this base class.
/// </summary>
/// <remarks>
/// This method cannot be overridden.
/// </remarks>
/// <param name="stream">The stream with the request payload.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream)
{
// read stream into memory
LogInfo("reading stream body");
string snsEventBody;
using (var reader = new StreamReader(stream)) {
snsEventBody = reader.ReadToEnd();
}
var stopwatch = Stopwatch.StartNew();
var metrics = new List <LambdaMetric>();
// process received sns record (there is only ever one)
try {
// sns event deserialization
LogInfo("deserializing SNS event");
try {
var snsEvent = LambdaSerializer.Deserialize <SNSEvent>(snsEventBody);
_currentRecord = snsEvent.Records.First().Sns;
// message deserialization
LogInfo("deserializing message");
var message = Deserialize(CurrentRecord.Message);
// process message
LogInfo("processing message");
await ProcessMessageAsync(message);
// record successful processing metrics
stopwatch.Stop();
var now = DateTimeOffset.UtcNow;
metrics.Add(("MessageSuccess.Count", 1, LambdaMetricUnit.Count));
metrics.Add(("MessageSuccess.Latency", stopwatch.Elapsed.TotalMilliseconds, LambdaMetricUnit.Milliseconds));
metrics.Add(("MessageSuccess.Lifespan", (now - CurrentRecord.GetLifespanTimestamp()).TotalSeconds, LambdaMetricUnit.Seconds));
return("Ok".ToStream());
} catch (Exception e) {
LogError(e);
try {
// attempt to send failed message to the dead-letter queue
await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.SQS, LambdaSerializer.Serialize(snsEventBody), e);
// record failed processing metrics
metrics.Add(("MessageDead.Count", 1, LambdaMetricUnit.Count));
} catch {
// NOTE (2020-04-22, bjorg): since the message could not be sent to the dead-letter queue,
// the next best action is to let Lambda retry it; unfortunately, there is no way
// of knowing how many attempts have occurred already.
// unable to forward message to dead-letter queue; report failure to lambda so it can retry
metrics.Add(("MessageFailed.Count", 1, LambdaMetricUnit.Count));
throw;
}
return($"ERROR: {e.Message}".ToStream());
}
} finally {
_currentRecord = null;
LogMetric(metrics);
}
}
private Exception Abort(AcknowledgeAction acknowledgeAction)
=> Abort(new APIGatewayProxyResponse {
StatusCode = 200,
Body = LambdaSerializer.Serialize(acknowledgeAction),
Headers = new Dictionary <string, string> {
["ContentType"] = "application/json"
}
});
public override sealed async Task <BotResponse> ProcessMessageAsync(BotRequest request)
{
// check if there is a state object to load
State = !string.IsNullOrEmpty(request.Bot?.InternalState)
? LambdaSerializer.Deserialize <TState>(request.Bot.InternalState)
: new TState();
LogInfo($"Starting State:\n{LambdaSerializer.Serialize(State)}");
// dispatch to specific method based on request command
BotResponse response;
switch (request.Command)
{
case BotCommand.GetBuild:
// bot configuration request
response = new BotResponse {
BotBuild = await GetBuildAsync()
};
break;
case BotCommand.GetAction:
// bot action request
try {
// capture request fields for easy access
GameSession = request.Session;
Bot = request.Bot;
GameClient = new LambdaRobotsGameClient(GameSession.ApiUrl, Bot.Id, HttpClient);
// initialize a default empty action
_action = new GetActionResponse();
// get bot action
await GetActionAsync();
// generate response
_action.BotState = LambdaSerializer.Serialize(State);
response = new BotResponse {
BotAction = _action,
};
} finally {
Bot = null;
GameClient = null;
}
break;
default:
// unrecognized request
throw new ApplicationException($"unexpected request: '{request.Command}'");
}
// log response and return
LogInfo($"Final State:\n{LambdaSerializer.Serialize(State)}");
return(response);
}
//--- Methods ---
/// <summary>
/// The <see cref="ProcessMessageStreamAsync(Stream)"/> method is overridden to
/// provide specific behavior for this base class.
/// </summary>
/// <remarks>
/// This method cannot be overridden.
/// </remarks>
/// <param name="stream">The stream with the request payload.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream)
{
var schedule = LambdaSerializer.Deserialize <LambdaScheduleEvent>(stream);
LogInfo($"received schedule event '{schedule.Name ?? schedule.Id}'");
await ProcessEventAsync(schedule);
return("Ok".ToStream());
}
protected async Task <bool> Respond(SlackRequest request, SlackResponse response)
{
var httpResponse = await HttpClient.SendAsync(new HttpRequestMessage {
RequestUri = new Uri(request.ResponseUrl),
Method = HttpMethod.Post,
Content = new StringContent(LambdaSerializer.Serialize(response))
});
return(httpResponse.StatusCode == HttpStatusCode.OK);
}
public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream)
{
// sns event deserialization
LogInfo("reading message stream");
SlackRequest request;
try {
request = LambdaSerializer.Deserialize <SlackRequest>(stream);
} catch (Exception e) {
LogError(e, "failed during Slack request deserialization");
return($"ERROR: {e.Message}".ToStream());
}
// capture standard output and error output so we can send it to slack instead
using (var consoleOutWriter = new StringWriter())
using (var consoleErrorWriter = new StringWriter()) {
var consoleOutOriginal = Console.Out;
var consoleErrorOriginal = Console.Error;
try {
// redirect the console output and error streams so we can emit them later to slack
Console.SetOut(consoleOutWriter);
Console.SetError(consoleErrorWriter);
// validate the slack token (assuming one was configured)
if (!(_slackVerificationToken?.Equals(request.Token) ?? true))
{
throw new SlackVerificationTokenMismatchException();
}
// handle slack request
await ProcessSlackRequestAsync(request);
} catch (Exception e) {
LogError(e);
Console.Error.WriteLine(e);
} finally {
Console.SetOut(consoleOutOriginal);
Console.SetError(consoleErrorOriginal);
}
// send console output to slack as an in_channel response
var output = consoleOutWriter.ToString();
if (output.Length > 0)
{
await RespondInChannel(request, output);
}
// send console error to slack as an ephemeral response (only visible to the requesting user)
var error = consoleErrorWriter.ToString();
if (error.Length > 0)
{
await RespondEphemeral(request, error);
}
}
return("Ok".ToStream());
}
//--- Methods ---
/// <summary>
/// The <see cref="ProcessMessageStreamAsync(Stream)"/> deserializes the request stream into
/// a <typeparamref name="TRequest"/> instance and invokes the <see cref="ProcessMessageAsync(TRequest)"/> method.
/// </summary>
/// <remarks>
/// This method is <c>sealed</c> and cannot be overridden.
/// </remarks>
/// <param name="stream">The stream with the request payload.</param>
/// <returns>The task object representing the asynchronous operation.</returns>
public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream)
{
var request = LambdaSerializer.Deserialize <TRequest>(stream);
var response = await ProcessMessageAsync(request);
var responseStream = new MemoryStream();
LambdaSerializer.Serialize(response, responseStream);
responseStream.Position = 0;
return(responseStream);
}
private Task NotifyAsync <T>(string userId, string channelId, T notification, int delay = 0) where T : ANotification
=> _sqsClient.SendMessageAsync(new Amazon.SQS.Model.SendMessageRequest {
MessageBody = LambdaSerializer.Serialize(new BroadcastMessage {
UserId = userId,
ChannelId = channelId,
Payload = LambdaSerializer.Serialize(notification)
}
),
QueueUrl = _notifyQueueUrl,
DelaySeconds = delay
});
private async Task WriteResponse(
CloudFormationResourceRequest <TProperties> rawRequest,
CloudFormationResourceResponse <TAttributes> rawResponse
)
{
Exception exception = null;
var backoff = TimeSpan.FromMilliseconds(100);
// write response to pre-signed S3 URL
for (var i = 0; i < MAX_SEND_ATTEMPTS; ++i)
{
try {
if (rawRequest.ResponseURL == null)
{
throw new InvalidOperationException("ResponseURL is missing");
}
var httpResponse = await HttpClient.SendAsync(new HttpRequestMessage {
RequestUri = new Uri(rawRequest.ResponseURL),
Method = HttpMethod.Put,
Content = new ByteArrayContent(Encoding.UTF8.GetBytes(LambdaSerializer.Serialize(rawResponse)))
});
if (httpResponse.StatusCode != HttpStatusCode.OK)
{
throw new LambdaCustomResourceException(
"PUT operation to pre-signed S3 URL failed with status code: {0} [{1} {2}] = {3}",
httpResponse.StatusCode,
rawRequest.RequestType,
rawRequest.ResourceType ?? "<MISSING>",
await httpResponse.Content.ReadAsStringAsync()
);
}
return;
} catch (InvalidOperationException e) {
exception = e;
break;
} catch (Exception e) {
exception = e;
LogErrorAsWarning(e, "writing response to pre-signed S3 URL failed");
await Task.Delay(backoff);
backoff = TimeSpan.FromSeconds(backoff.TotalSeconds * 2);
}
}
if (exception == null)
{
exception = new ShouldNeverHappenException($"ALambdaCustomResourceFunction.WriteResponse failed w/o an explicit");
}
// max attempts have been reached; fail permanently and record the failed request for playback
LogError(exception);
await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.CloudFormation, LambdaSerializer.Serialize(rawRequest), exception);
}
public void ValidObjectShouldSerialize()
{
var serializer = new LambdaSerializer();
var stream = new MemoryStream();
var obj = new TestRequest()
{
Test = "Test"
};
serializer.Serialize(obj, stream);
stream.Position = 0;
Assert.Equal("{\"Test\":\"Test\"}", new StreamReader(stream).ReadToEnd());
}
public void InvalidJsonShouldNotDeserialize()
{
var serializer = new LambdaSerializer();
var stream = new MemoryStream();
var json = "123";
var sw = new StreamWriter(stream);
sw.Write(json);
sw.Flush();
stream.Position = 0;
Assert.Throws(typeof(LambdaException), () => serializer.Deserialize <TestRequest>(stream));
}
public override async Task ProcessMessageAsync(Message message)
{
await Task.WhenAll(
_sqsClient.SendMessageAsync(_queueUrl, LambdaSerializer.Serialize(new Message {
Counter = message.Counter + 1
})),
_sqsClient.SendMessageAsync(_queueUrl, LambdaSerializer.Serialize(new Message {
Counter = message.Counter + 2
}))
);
throw new Exception("oops!");
}
public override async Task <FunctionResponse> ProcessMessageAsync(CloudWatchEvent <EventDetails> request)
{
LogInfo($"Version = {request.Version}");
LogInfo($"Account = {request.Account}");
LogInfo($"Region = {request.Region}");
LogInfo($"Detail = {LambdaSerializer.Serialize(request.Detail)}");
LogInfo($"DetailType = {request.DetailType}");
LogInfo($"Source = {request.Source}");
LogInfo($"Time = {request.Time}");
LogInfo($"Id = {request.Id}");
LogInfo($"Resources = [{string.Join(",", request.Resources ?? Enumerable.Empty<string>())}]");
LogInfo($"Latency = {DateTime.UtcNow - request.Time}");
return(new FunctionResponse());
}
public override async Task <MacroResponse> ProcessMessageAsync(MacroRequest request)
{
LogInfo($"AwsRegion = {request.Region}");
LogInfo($"AccountID = {request.AccountId}");
LogInfo($"Fragment = {LambdaSerializer.Serialize(request.Fragment)}");
LogInfo($"TransformID = {request.TransformId}");
LogInfo($"Params = {LambdaSerializer.Serialize(request.Params)}");
LogInfo($"RequestID = {request.RequestId}");
LogInfo($"TemplateParameterValues = {LambdaSerializer.Serialize(request.TemplateParameterValues)}");
// macro for string operations
try {
if (!request.Params.TryGetValue("Value", out var value))
{
throw new ArgumentException("missing parameter: 'Value");
}
if (!(value is string text))
{
throw new ArgumentException("parameter 'Value' must be a string");
}
string result;
switch (request.TransformId)
{
case "StringToUpper":
result = text.ToUpper();
break;
case "StringToLower":
result = text.ToLower();
break;
default:
throw new NotSupportedException($"requested operation is not supported: '{request.TransformId}'");
}
// return successful response
return(new MacroResponse {
RequestId = request.RequestId,
Status = "SUCCESS",
Fragment = result
});
} catch (Exception e) {
// an error occurred
return(new MacroResponse {
RequestId = request.RequestId,
Status = $"ERROR: {e.Message}"
});
}
}
// [Route("$connect")]
public async Task <APIGatewayProxyResponse> OpenConnectionAsync(APIGatewayProxyRequest request)
{
LogInfo($"Connected: {request.RequestContext.ConnectionId}");
// verify client authorization
string headerBase64Json = null;
if (!(request.QueryStringParameters?.TryGetValue("header", out headerBase64Json) ?? false))
{
// reject connection request
return(new APIGatewayProxyResponse {
StatusCode = (int)HttpStatusCode.Unauthorized
});
}
ConnectionHeader header;
try {
var headerJson = Encoding.UTF8.GetString(Convert.FromBase64String(headerBase64Json));
header = LambdaSerializer.Deserialize <ConnectionHeader>(headerJson);
} catch {
// reject connection request
return(new APIGatewayProxyResponse {
StatusCode = (int)HttpStatusCode.BadRequest
});
}
if (
(header.Host != request.RequestContext.DomainName) ||
(header.ApiKey != _clientApiKey) ||
!Guid.TryParse(header.Id, out _)
)
{
// reject connection request
return(new APIGatewayProxyResponse {
StatusCode = (int)HttpStatusCode.Forbidden
});
}
// create new connection record
await _dataTable.CreateConnectionRecordAsync(new ConnectionRecord {
ConnectionId = request.RequestContext.ConnectionId,
State = ConnectionState.New,
ApplicationId = header.Id,
Bearer = request.RequestContext.Authorizer?.Claims
});
return(new APIGatewayProxyResponse {
StatusCode = (int)HttpStatusCode.OK
});
}
public void ValidJsonShouldDeserialize()
{
var serializer = new LambdaSerializer();
var stream = new MemoryStream();
var json = "{\"Test\":\"Test\"}";
var sw = new StreamWriter(stream);
sw.Write(json);
sw.Flush();
stream.Position = 0;
var d = serializer.Deserialize <TestRequest>(stream);
Assert.Equal("Test", d.Test);
}
public override async Task ProcessEventAsync(EventDetails details)
{
var request = CurrentEvent;
LogInfo($"Version = {request.Version}");
LogInfo($"Account = {request.Account}");
LogInfo($"Region = {request.Region}");
LogInfo($"Detail = {LambdaSerializer.Serialize(details)}");
LogInfo($"DetailType = {request.DetailType}");
LogInfo($"Source = {request.Source}");
LogInfo($"Time = {request.Time}");
LogInfo($"Id = {request.Id}");
LogInfo($"Resources = [{string.Join(",", request.Resources ?? Enumerable.Empty<string>())}]");
LogInfo($"Latency = {DateTime.UtcNow - request.Time}");
}
//--- Methods ---
/// <summary>
/// The <see cref="InitializeEpilogueAsync()"/> method is invoked to complet the initialization of the
/// Lambda function. This is the last of three methods that are invoked to initialize the Lambda function.
/// </summary>
/// <returns>The task object representing the asynchronous operation.</returns>
protected override async Task InitializeEpilogueAsync()
{
await base.InitializeEpilogueAsync();
// NOTE (2019-04-15, bjorg): we initialize the invocation target directory after the function environment
// initialization so that 'CreateInvocationTargetInstance()' can access the environment variables if need be.
// read optional api-gateway-mappings file
_directory = new ApiGatewayInvocationTargetDirectory(CreateInvocationTargetInstance, LambdaSerializer);
if (File.Exists("api-mappings.json"))
{
var mappings = LambdaSerializer.Deserialize <ApiGatewayInvocationMappings>(File.ReadAllText("api-mappings.json"));
if (mappings.Mappings == null)
{
throw new InvalidDataException("missing 'Mappings' property in 'api-mappings.json' file");
}
foreach (var mapping in mappings.Mappings)
{
if (mapping.Method == null)
{
throw new InvalidDataException("missing 'Method' property in 'api-mappings.json' file");
}
if (mapping.RestApi != null)
{
LogInfo($"Mapping REST API '{mapping.RestApi}' to {mapping.Method}");
_directory.Add(mapping.RestApi, mapping.Method);
}
else if (mapping.WebSocket != null)
{
LogInfo($"Mapping WebSocket route '{mapping.WebSocket}' to {mapping.Method}");
_directory.Add(mapping.WebSocket, mapping.Method);
}
else
{
throw new InvalidDataException("missing 'RestApi/WebSocket' property in 'api-mappings.json' file");
}
}
}
}
public override async Task ProcessEventAsync(BitcoinPriceEvent message)
{
// TO-DO: add business logic
LogInfo($"Received: {LambdaSerializer.Serialize(message)}");
}
public override async Task <KinesisFirehoseResponse> ProcessMessageAsync(KinesisFirehoseEvent request)
{
// NOTE (2018-12-11, bjorg): this function is responsible for error logs parsing; therefore, it CANNOT error out itself;
// instead, it must rely on aggressive exception handling and redirect those message where appropriate.
ResetMetrics();
var response = new KinesisFirehoseResponse {
Records = new List <KinesisFirehoseResponse.FirehoseRecord>()
};
_approximateResponseSize = 0;
var reingestedCount = 0;
try {
for (var recordIndex = 0; recordIndex < request.Records.Count; ++recordIndex)
{
var record = request.Records[recordIndex];
try {
var logEventsMessage = ConvertRecordToLogEventsMessage(record);
// validate log event
if (
(logEventsMessage.LogGroup is null) ||
(logEventsMessage.MessageType is null) ||
(logEventsMessage.LogEvents is null)
)
{
LogWarn("invalid log events message (record-id: {0})", record.RecordId);
RecordFailed(record);
continue;
}
// skip log event from own module
if ((Info.FunctionName != null) && logEventsMessage.LogGroup.Contains(Info.FunctionName))
{
LogInfo($"skipping log events message from own event log (record-id: {record.RecordId})");
RecordDropped(record);
continue;
}
// skip control log event
if (logEventsMessage.MessageType == "CONTROL_MESSAGE")
{
LogInfo($"skipping control log events message (record-id: {record.RecordId})");
RecordDropped(record);
continue;
}
// check if this log event is expected
if (logEventsMessage.MessageType != "DATA_MESSAGE")
{
LogWarn("unknown log events message type '{1}' (record-id: {0})", record.RecordId, logEventsMessage.MessageType);
RecordFailed(record);
continue;
}
// check if log group belongs to a Lambda function
OwnerMetaData?owner;
if (logEventsMessage.LogGroup.StartsWith(LAMBDA_LOG_GROUP_PREFIX, StringComparison.Ordinal))
{
// use CloudWatch log name to identify owner of the log event
var functionId = logEventsMessage.LogGroup.Substring(LAMBDA_LOG_GROUP_PREFIX.Length);
owner = await GetOwnerMetaDataAsync($"F:{functionId}");
}
else
{
owner = await GetOwnerMetaDataAsync($"L:{logEventsMessage.LogGroup}");
}
// check if owner record exists
if (owner == null)
{
LogInfo($"skipping log events message for unknown log-group (record-id: {record.RecordId}, log-group: {logEventsMessage.LogGroup})");
RecordDropped(record);
continue;
}
// process entries in log event
_convertedLogEvents.Clear();
var success = true;
for (var logEventIndex = 0; logEventIndex < logEventsMessage.LogEvents.Count; ++logEventIndex)
{
var logEvent = logEventsMessage.LogEvents[logEventIndex];
try {
await Logic.ProgressLogEntryAsync(
owner,
logEvent.Message,
DateTimeOffset.FromUnixTimeMilliseconds(logEvent.Timestamp)
);
} catch (Exception e) {
if (owner.FunctionId != null)
{
LogError(e, "log event [{1}] processing failed (function-id: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, logEvent.Message ?? "<null>", owner.FunctionId);
}
else if (owner.AppId != null)
{
LogError(e, "log event [{1}] processing failed (app-id: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, logEvent.Message ?? "<null>", owner.AppId);
}
else
{
LogError(e, "log event [{1}] processing failed (log-group: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, logEvent.Message ?? "<null>", logEventsMessage.LogGroup);
}
// mark this log events message as failed and stop processing more log events
success = false;
break;
}
}
// check outcome of processing log events message
if (success)
{
// check if any log events were converted
if (_convertedLogEvents.Any())
{
// calculate size of the converted log events
var convertedLogEventsSize = _convertedLogEvents.Sum(convertedLogEvent => convertedLogEvent.SerializedByteCount);
if ((_approximateResponseSize + convertedLogEventsSize) > RESPONSE_SIZE_LIMIT)
{
// check if response size was exceeded on first record
if (response.Records.Count == 0)
{
// skip record since it's the first record and we cannot serialize it due to response size limits
LogWarn("record too large to convert (record-id: {0})", record.RecordId);
RecordFailed(record);
}
else
{
LogInfo($"reached Lambda response limit (response: {_approximateResponseSize:N0}, limit: {RESPONSE_SIZE_LIMIT:N0})");
// reingest remaining records since the response will be too large otherwise
var firehoseDeliveryStream = request.DeliveryStreamArn.Split('/').Last();
var reingestedRecords = request.Records.Skip(recordIndex).Select(record => new Record {
Data = new MemoryStream(Convert.FromBase64String(record.Base64EncodedData))
}).ToList();
await FirehoseClient.PutRecordBatchAsync(firehoseDeliveryStream, reingestedRecords);
reingestedCount += reingestedRecords.Count;
// drop reingested records
for (; recordIndex < request.Records.Count; ++recordIndex)
{
var droppedRecord = request.Records[recordIndex];
LogInfo($"reingested record (record-id: {droppedRecord.RecordId}");
RecordDropped(droppedRecord);
}
}
}
else
{
_approximateResponseSize += convertedLogEventsSize;
// measure how long it took to successfully process the first CloudWatch Log event in the record
if (logEventsMessage.LogEvents.Any())
{
try {
var timestamp = DateTimeOffset.FromUnixTimeMilliseconds(logEventsMessage.LogEvents.First().Timestamp);
LogMetric("LogEvent.Latency", (DateTimeOffset.UtcNow - timestamp).TotalMilliseconds, LambdaMetricUnit.Milliseconds);
} catch (Exception e) {
LogError(e, "report log event latency failed");
}
}
// emit LambdaSharp events from converted log entries
await ProcessConvertedLogEntriesAsync();
LogInfo($"finished converting log events (converted {_convertedLogEvents.Count:N0}, skipped {logEventsMessage.LogEvents.Count - _convertedLogEvents.Count:N0}, record-id: {record.RecordId})");
RecordSuccess(record, _convertedLogEvents.Aggregate("", (accumulator, convertedLogEvent) => accumulator + convertedLogEvent.Json + "\n"));
}
}
else
{
LogInfo($"dropped record (record-id: {record.RecordId}");
RecordDropped(record);
}
}
else
{
// nothing to log since error was already logged
RecordFailed(record);
}
} catch (Exception e) {
LogError(e, "record failed (record-id: {0})", record.RecordId);
RecordFailed(record);
}
}
// show processing outcome
var okResponsesCount = response.Records.Count(r => r.Result == KinesisFirehoseResponse.TRANSFORMED_STATE_OK);
var failedResponsesCount = response.Records.Count(r => r.Result == KinesisFirehoseResponse.TRANSFORMED_STATE_PROCESSINGFAILED);
var droppedResponsesCount = response.Records.Count(r => r.Result == KinesisFirehoseResponse.TRANSFORMED_STATE_DROPPED) - reingestedCount;
LogInfo($"processed {request.Records.Count:N0} records (success: {okResponsesCount}, failed: {failedResponsesCount:N0}, dropped: {droppedResponsesCount:N0}, reingested: {reingestedCount:N0})");
} finally {
// NOTE (2020-04-21, bjorg): we don't expect this to fail; but since it's done at the end of the processing function, we
// need to make sure it never fails; otherwise, the Kinesis stream processing is interrupted.
try {
ReportMetrics();
} catch (Exception e) {
LogError(e, "report metrics failed");
}
// send accumulated events
try {
PurgeEventEntries();
} catch (Exception exception) {
// NOTE (2020-12-28, bjorg): don't use LogError() as it will eventually send an event, since there
// is no other reporting mechanism for LambdaSharp.Core otherwise.
Provider.Log($"EXCEPTION: {exception}\n");
}
}
return(response);
// local functions
LogEventsMessage ConvertRecordToLogEventsMessage(KinesisFirehoseEvent.FirehoseRecord record)
{
// deserialize kinesis record into a CloudWatch Log events message
using var sourceStream = new MemoryStream(Convert.FromBase64String(record.Base64EncodedData));
using var recordData = new MemoryStream();
using (var gzip = new GZipStream(sourceStream, CompressionMode.Decompress)) {
gzip.CopyTo(recordData);
recordData.Position = 0;
}
return(LambdaSerializer.Deserialize <LogEventsMessage>(Encoding.UTF8.GetString(recordData.ToArray())));
}
void RecordSuccess(KinesisFirehoseEvent.FirehoseRecord record, string data)
=> response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord {
RecordId = record.RecordId,
Result = KinesisFirehoseResponse.TRANSFORMED_STATE_OK,
Base64EncodedData = Convert.ToBase64String(Encoding.UTF8.GetBytes(data))
});
void RecordFailed(KinesisFirehoseEvent.FirehoseRecord record)
=> response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord {
RecordId = record.RecordId,
Result = KinesisFirehoseResponse.TRANSFORMED_STATE_PROCESSINGFAILED
});
void RecordDropped(KinesisFirehoseEvent.FirehoseRecord record)
=> response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord {
RecordId = record.RecordId,
Result = KinesisFirehoseResponse.TRANSFORMED_STATE_DROPPED
});
}
public override async Task <APIGatewayHttpApiV2ProxyResponse> ProcessMessageAsync(APIGatewayHttpApiV2ProxyRequest request)
{
LogInfo($"Message received at {request.RequestContext.Http.Method}:{request.RawPath}?{request.RawQueryString}");
// validate invocation method
if (request.RequestContext.Http.Method != "POST")
{
LogWarn("Unsupported request method {0}", request.RequestContext.Http.Method);
return(BadRequestResponse());
}
// validate request token
if (
!request.QueryStringParameters.TryGetValue("token", out var token) ||
(token != _httpApiToken)
)
{
LogWarn("Missing or invalid request token");
return(BadRequestResponse());
}
// validate request websocket
if (
!request.QueryStringParameters.TryGetValue("ws", out var connectionId) ||
string.IsNullOrEmpty(connectionId)
)
{
LogWarn("Invalid websocket connection id");
return(BadRequestResponse());
}
// validate request id
if (
!request.QueryStringParameters.TryGetValue("rid", out var requestId) ||
string.IsNullOrEmpty(requestId)
)
{
LogWarn("Invalid request id");
return(BadRequestResponse());
}
// check if request is a subscription confirmation
var topicSubscription = LambdaSerializer.Deserialize <TopicSubscriptionPayload>(request.Body);
if (topicSubscription.Type == "SubscriptionConfirmation")
{
return(await ConfirmSubscription(connectionId, requestId, topicSubscription));
}
// validate SNS message
var snsMessage = LambdaSerializer.Deserialize <SNSEvent.SNSMessage>(request.Body);
if (snsMessage.Message == null)
{
LogWarn("Invalid SNS message received: {0}", request.Body);
return(BadRequestResponse());
}
// validate EventBridge event
var eventBridgeEvent = LambdaSerializer.Deserialize <EventBridgeventPayload>(snsMessage.Message);
if (
(eventBridgeEvent.Source == null) ||
(eventBridgeEvent.DetailType == null) ||
(eventBridgeEvent.Resources == null)
)
{
LogInfo("Invalid EventBridge event received: {0}", snsMessage.Message);
return(BadRequestResponse());
}
return(await DispatchEvent(connectionId, snsMessage.Message));
}
public override async Task <KinesisFirehoseResponse> ProcessMessageAsync(KinesisFirehoseEvent request)
{
// NOTE (2018-12-11, bjorg): this function is responsible for error logs parsing; therefore, it CANNOT error out itself;
// instead, it must rely on aggressive exception handling and redirect those message where appropriate.
ResetMetrics();
var response = new KinesisFirehoseResponse {
Records = new List <KinesisFirehoseResponse.FirehoseRecord>()
};
try {
foreach (var record in request.Records)
{
try {
// deserialize kinesis record into a CloudWatch Log event
LogEventsMessage logEvent;
using (var sourceStream = new MemoryStream(Convert.FromBase64String(record.Base64EncodedData)))
using (var destinationStream = new MemoryStream()) {
using (var gzip = new GZipStream(sourceStream, CompressionMode.Decompress)) {
gzip.CopyTo(destinationStream);
destinationStream.Position = 0;
}
logEvent = LambdaSerializer.Deserialize <LogEventsMessage>(Encoding.UTF8.GetString(destinationStream.ToArray()));
}
// validate log event
if (
(logEvent.LogGroup == null) ||
(logEvent.MessageType == null) ||
(logEvent.LogEvents == null)
)
{
LogWarn("invalid record (record-id: {0})", record.RecordId);
RecordFailed(record);
continue;
}
// skip log event from own module
if (logEvent.LogGroup.Contains(Info.FunctionName))
{
LogInfo("skipping event from own event log (record-id: {0})", record.RecordId);
RecordDropped(record);
continue;
}
// skip control log event
if (logEvent.MessageType == "CONTROL_MESSAGE")
{
LogInfo("skipping control message (record-id: {0})", record.RecordId);
RecordDropped(record);
continue;
}
// check if this log event is expected
if (logEvent.MessageType != "DATA_MESSAGE")
{
LogWarn("unknown message type '{1}' (record-id: {0})", record.RecordId, logEvent.MessageType);
RecordFailed(record);
continue;
}
// check if log group belongs to a Lambda function
OwnerMetaData?owner;
if (logEvent.LogGroup.StartsWith(LAMBDA_LOG_GROUP_PREFIX, StringComparison.Ordinal))
{
// use CloudWatch log name to identify owner of the log event
var functionId = logEvent.LogGroup.Substring(LAMBDA_LOG_GROUP_PREFIX.Length);
owner = await GetOwnerMetaDataAsync($"F:{functionId}");
}
else
{
owner = await GetOwnerMetaDataAsync($"L:{logEvent.LogGroup}");
}
// check if owner record exists
if (owner == null)
{
LogInfo("skipping record for non-registered log-group (record-id: {0}, log-group: {1})", record.RecordId, logEvent.LogGroup);
RecordDropped(record);
continue;
}
// process entries in log event
_convertedRecords.Clear();
var success = true;
var logEventIndex = -1;
foreach (var entry in logEvent.LogEvents)
{
++logEventIndex;
try {
await Logic.ProgressLogEntryAsync(
owner,
entry.Message,
DateTimeOffset.FromUnixTimeMilliseconds(entry.Timestamp)
);
} catch (Exception e) {
if (owner.FunctionId != null)
{
LogError(e, "log event entry [{1}] processing failed (function-id: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, entry.Message, owner.FunctionId);
}
else if (owner.AppId != null)
{
LogError(e, "log event entry [{1}] processing failed (app-id: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, entry.Message, owner.AppId);
}
else
{
LogError(e, "log event entry [{1}] processing failed (log-group: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, entry.Message, logEvent.LogGroup);
}
success = false;
break;
}
}
// record how long it took to process the CloudWatch Log event
if (logEvent.LogEvents.Any())
{
try {
var timestamp = DateTimeOffset.FromUnixTimeMilliseconds(logEvent.LogEvents.First().Timestamp);
LogMetric("LogEvent.Latency", (DateTimeOffset.UtcNow - timestamp).TotalMilliseconds, LambdaMetricUnit.Milliseconds);
} catch (Exception e) {
LogError(e, "report log event latency failed");
}
}
// record outcome
if (success)
{
if (_convertedRecords.Any())
{
LogInfo($"finished log events record (converted {_convertedRecords.Count:N0}, skipped {logEvent.LogEvents.Count - _convertedRecords.Count:N0}, record-id: {record.RecordId})");
RecordSuccess(record, _convertedRecords.Aggregate("", (accumulator, convertedRecord) => accumulator + convertedRecord + "\n"));
}
else
{
LogInfo($"dropped record (record-id: {record.RecordId}");
RecordDropped(record);
}
}
else
{
// nothing to log since error was already logged
RecordFailed(record);
}
} catch (Exception e) {
LogError(e, "record failed (record-id: {0})", record.RecordId);
RecordFailed(record);
}
}
} finally {
// NOTE (2020-04-21, bjorg): we don't expect this to fail; but since it's done at the end of the processing function, we
// need to make sure it never fails; otherwise, the Kinesis stream processing is interrupted.
try {
ReportMetrics();
} catch (Exception e) {
LogError(e, "report metrics failed");
}
// send accumulated events
try {
PurgeEventEntries();
} catch (Exception exception) {
Provider.Log($"EXCEPTION: {exception}\n");
}
}
return(response);
// local functions
void RecordSuccess(KinesisFirehoseEvent.FirehoseRecord record, string data)
=> response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord {
RecordId = record.RecordId,
Result = KinesisFirehoseResponse.TRANSFORMED_STATE_OK,
Base64EncodedData = Convert.ToBase64String(Encoding.UTF8.GetBytes(data))
});
void RecordFailed(KinesisFirehoseEvent.FirehoseRecord record)
=> response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord {
RecordId = record.RecordId,
Result = KinesisFirehoseResponse.TRANSFORMED_STATE_PROCESSINGFAILED
});
void RecordDropped(KinesisFirehoseEvent.FirehoseRecord record)
=> response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord {
RecordId = record.RecordId,
Result = KinesisFirehoseResponse.TRANSFORMED_STATE_DROPPED
});
}
//--- Methods ---
/// <summary>
/// The <see cref="Deserialize(string)"/> method converts the SNS topic message from string to a typed instance.
/// </summary>
/// <remarks>
/// This method invokes <see cref="Amazon.Lambda.Core.ILambdaSerializer.Deserialize{TMessage}(Stream)"/> to convert the SNS topic message string
/// into a <paramtyperef name="TMessage"/> instance. Override this method to provide a custom message deserialization implementation.
/// </remarks>
/// <param name="body">The SNS topic message.</param>
/// <returns>The deserialized SNS topic message.</returns>
public virtual TMessage Deserialize(string body) => LambdaSerializer.Deserialize <TMessage>(body);
public override async Task ProcessMessageAsync(SNSEvent.SNSMessage sns)
{
var evt = LambdaSerializer.Deserialize <Event>(sns.Message);
LogInfo($"Received {evt.EventType}: {evt.Message}");
}
public override async Task <APIGatewayHttpApiV2ProxyResponse> ProcessMessageAsync(APIGatewayHttpApiV2ProxyRequest request)
{
LogInfo($"Message received at {request.RequestContext.Http.Method}:{request.RawPath}?{request.RawQueryString}");
// validate invocation method
if (request.RequestContext.Http.Method != "POST")
{
LogInfo("Unsupported request method {0}", request.RequestContext.Http.Method);
return(BadRequest());
}
// validate request token
if (
!request.QueryStringParameters.TryGetValue("token", out var token) ||
(token != _httpApiToken)
)
{
LogInfo("Missing or invalid request token");
return(BadRequest());
}
// validate request websocket
if (
!request.QueryStringParameters.TryGetValue("ws", out var connectionId) ||
string.IsNullOrEmpty(connectionId)
)
{
LogInfo("Invalid websocket connection id");
return(BadRequest());
}
// validate request id
if (
!request.QueryStringParameters.TryGetValue("rid", out var requestId) ||
string.IsNullOrEmpty(requestId)
)
{
LogInfo("Invalid request id");
return(BadRequest());
}
// check if request is a subscription confirmation
var topicSubscription = LambdaSerializer.Deserialize <TopicSubscriptionPayload>(request.Body);
if (topicSubscription.Type == "SubscriptionConfirmation")
{
// confirm it's for the expected topic ARN
if (topicSubscription.TopicArn != _eventTopicArn)
{
LogWarn("Wrong Topic ARN for subscription confirmation (Expected: {0}, Received: {1})", _eventTopicArn, topicSubscription.TopicArn);
return(BadRequest());
}
// confirm subscription
await HttpClient.GetAsync(topicSubscription.SubscribeURL);
// send welcome action to websocket connection
await SendMessageToConnection(new AcknowledgeAction {
RequestId = requestId,
Status = "Ok"
}, connectionId);
return(Success("Confirmed"));
}
// validate SNS message
var snsMessage = LambdaSerializer.Deserialize <SNSEvent.SNSMessage>(request.Body);
if (snsMessage.Message == null)
{
LogWarn("Invalid SNS message received: {0}", request.Body);
return(BadRequest());
}
// validate CloudWatch event
var cloudWatchEvent = LambdaSerializer.Deserialize <CloudWatchEventPayload>(snsMessage.Message);
if (
(cloudWatchEvent.Source == null) ||
(cloudWatchEvent.DetailType == null) ||
(cloudWatchEvent.Resources == null)
)
{
LogInfo("Invalid CloudWatch event received: {0}", snsMessage.Message);
return(BadRequest());
}
// check if the keep-alive event was received
if (
(cloudWatchEvent.Source == "aws.events") &&
(cloudWatchEvent.DetailType == "Scheduled Event") &&
(cloudWatchEvent.Resources.Count == 1) &&
(cloudWatchEvent.Resources[0] == _keepAliveRuleArn)
)
{
// send keep-alive action to websocket connection
await SendMessageToConnection(new KeepAliveAction(), connectionId);
return(Success("Ok"));
}
// determine what rules are matching
JObject evt;
try {
evt = JObject.Parse(snsMessage.Message);
} catch (Exception e) {
LogError(e, "invalid message");
return(BadRequest());
}
var rules = await _dataTable.GetConnectionRulesAsync(connectionId);
var matchedRules = rules
.Where(rule => {
try {
var pattern = JObject.Parse(rule.Pattern);
return(EventPatternMatcher.IsMatch(evt, pattern));
} catch (Exception e) {
LogError(e, "invalid event pattern: {0}", rule.Pattern);
return(false);
}
}).Select(rule => rule.Rule)
.ToList();
if (matchedRules.Any())
{
await SendMessageToConnection(
new EventAction {
Rules = matchedRules,
Source = cloudWatchEvent.Source,
Type = cloudWatchEvent.DetailType,
Event = snsMessage.Message
},
connectionId
);
}
return(Success("Ok"));
// local functions
APIGatewayHttpApiV2ProxyResponse Success(string message)
=> new APIGatewayHttpApiV2ProxyResponse
{
Body = message,
Headers = new Dictionary <string, string> {
["Content-Type"] = "text/plain"
},
StatusCode = (int)HttpStatusCode.OK
};
APIGatewayHttpApiV2ProxyResponse BadRequest()
=> new APIGatewayHttpApiV2ProxyResponse
{
Body = "Bad Request",
Headers = new Dictionary <string, string> {
["Content-Type"] = "text/plain"
},
StatusCode = (int)HttpStatusCode.BadRequest
};
}
private async Task <APIGatewayHttpApiV2ProxyResponse> DispatchEvent(string connectionId, string message)
{
// validate EventBridge event
var eventBridgeEvent = LambdaSerializer.Deserialize <EventBridgeventPayload>(message);
if (
(eventBridgeEvent.Source == null) ||
(eventBridgeEvent.DetailType == null) ||
(eventBridgeEvent.Resources == null)
)
{
LogInfo("Invalid EventBridge event received: {0}", message);
return(BadRequestResponse());
}
// check if the keep-alive event was received
if (
(eventBridgeEvent.Source == "aws.events") &&
(eventBridgeEvent.DetailType == "Scheduled Event") &&
(eventBridgeEvent.Resources?.Count == 1) &&
(eventBridgeEvent.Resources[0] == _keepAliveRuleArn)
)
{
// retrieve connection record
var connection = await _dataTable.GetConnectionRecordAsync(connectionId);
if (connection == null)
{
return(SuccessResponse("Gone"));
}
if (connection.State != ConnectionState.Open)
{
return(SuccessResponse("Ignored"));
}
// send keep-alive action to websocket connection
LogInfo("KeepAlive tick");
await SendActionToConnection(new KeepAliveAction(), connectionId);
return(SuccessResponse("Ok"));
}
// determine what rules are matching the event
JObject evt;
try {
evt = JObject.Parse(message);
} catch (Exception e) {
LogError(e, "invalid message");
return(BadRequestResponse());
}
var rules = await _dataTable.GetAllRuleRecordAsync(connectionId);
var matchedRules = rules
.Where(rule => {
try {
var pattern = JObject.Parse(rule.Pattern);
return(EventPatternMatcher.IsMatch(evt, pattern));
} catch (Exception e) {
LogError(e, "invalid event pattern: {0}", rule.Pattern);
return(false);
}
}).Select(rule => rule.Rule)
.ToList();
if (matchedRules.Any())
{
LogInfo($"Event matched {matchedRules.Count():N0} rules: {string.Join(", ", matchedRules)}");
await SendActionToConnection(
new EventAction {
Rules = matchedRules,
Source = eventBridgeEvent.Source,
Type = eventBridgeEvent.DetailType,
Event = message
},
connectionId
);
}
else
{
LogInfo("Event matched no rules");
}
return(SuccessResponse("Ok"));
}
