private async Task SendMessageToConnection(AnAction action, string connectionId) { var json = LambdaSerializer.Serialize <object>(action); if (DebugLoggingEnabled) { LogDebug($"Post to connection: {connectionId}\n{{0}}", json); } else { LogInfo($"Post to connection: {connectionId}"); } // attempt to send serialized message to connection var messageBytes = Encoding.UTF8.GetBytes(json); try { await _amaClient.PostToConnectionAsync(new PostToConnectionRequest { ConnectionId = connectionId, Data = new MemoryStream(messageBytes) }); } catch (AmazonServiceException e) when(e.StatusCode == System.Net.HttpStatusCode.Gone) { // HTTP Gone status code indicates the connection has been closed; nothing to do } catch (Exception e) { LogErrorAsWarning(e, "PostToConnectionAsync() failed on connection {0}", connectionId); } }
public override async Task <FunctionResponse> ProcessMessageAsync(FunctionRequest request) { LogInfo("Deserialized using {1}: {0}", LambdaSerializer.Serialize(request), LambdaSerializer.GetType().FullName); return(new FunctionResponse { Bar = request.Bar }); }
public async Task SendMessageAsync(Message request) { // enumerate open connections var connections = await _connections.GetAllRowsAsync(); LogInfo($"Found {connections.Count()} open connection(s)"); // attempt to send message on all open connections var messageBytes = Encoding.UTF8.GetBytes(LambdaSerializer.Serialize(new Message { From = request.From, Text = request.Text })); var outcomes = await Task.WhenAll(connections.Select(async(connectionId, index) => { LogInfo($"Post to connection {index}: {connectionId}"); try { await _amaClient.PostToConnectionAsync(new PostToConnectionRequest { ConnectionId = connectionId, Data = new MemoryStream(messageBytes) }); } catch (AmazonServiceException e) when(e.StatusCode == System.Net.HttpStatusCode.Gone) { LogInfo($"Deleting gone connection: {connectionId}"); await _connections.DeleteRowAsync(connectionId); return(false); } catch (Exception e) { LogErrorAsWarning(e, "PostToConnectionAsync() failed"); return(false); } return(true); })); LogInfo($"Data sent to {outcomes.Count(result => result)} connections"); }
private CloudFormationResourceRequest <TProperties> DeserializeStream(Stream stream) { // read stream into memory LogInfo("reading message stream"); string body; using (var reader = new StreamReader(stream)) { body = reader.ReadToEnd(); } // deserialize stream into a generic JSON object LogInfo("deserializing request"); var json = JsonConvert.DeserializeObject <JObject>(body); // determine if the custom resource request is wrapped in an SNS message // or if it is a direct invocation by the CloudFormation service if (json.TryGetValue("Records", out _)) { // deserialize SNS event LogInfo("deserializing SNS event"); var snsEvent = json.ToObject <SNSEvent>(); // extract message from SNS event LogInfo("deserializing message"); var messageBody = snsEvent.Records.First().Sns.Message; // deserialize message into a cloudformation request return(LambdaSerializer.Deserialize <CloudFormationResourceRequest <TProperties> >(messageBody)); } else { // deserialize generic JSON into a cloudformation request return(json.ToObject <CloudFormationResourceRequest <TProperties> >()); } }
/// <summary> /// The <see cref="ProcessMessageStreamAsync(Stream)"/> method is overridden to /// provide specific behavior for this base class. /// </summary> /// <remarks> /// This method cannot be overridden. /// </remarks> /// <param name="stream">The stream with the request payload.</param> /// <returns>The task object representing the asynchronous operation.</returns> public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream) { // read stream into memory LogInfo("reading stream body"); string snsEventBody; using (var reader = new StreamReader(stream)) { snsEventBody = reader.ReadToEnd(); } var stopwatch = Stopwatch.StartNew(); var metrics = new List <LambdaMetric>(); // process received sns record (there is only ever one) try { // sns event deserialization LogInfo("deserializing SNS event"); try { var snsEvent = LambdaSerializer.Deserialize <SNSEvent>(snsEventBody); _currentRecord = snsEvent.Records.First().Sns; // message deserialization LogInfo("deserializing message"); var message = Deserialize(CurrentRecord.Message); // process message LogInfo("processing message"); await ProcessMessageAsync(message); // record successful processing metrics stopwatch.Stop(); var now = DateTimeOffset.UtcNow; metrics.Add(("MessageSuccess.Count", 1, LambdaMetricUnit.Count)); metrics.Add(("MessageSuccess.Latency", stopwatch.Elapsed.TotalMilliseconds, LambdaMetricUnit.Milliseconds)); metrics.Add(("MessageSuccess.Lifespan", (now - CurrentRecord.GetLifespanTimestamp()).TotalSeconds, LambdaMetricUnit.Seconds)); return("Ok".ToStream()); } catch (Exception e) { LogError(e); try { // attempt to send failed message to the dead-letter queue await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.SQS, LambdaSerializer.Serialize(snsEventBody), e); // record failed processing metrics metrics.Add(("MessageDead.Count", 1, LambdaMetricUnit.Count)); } catch { // NOTE (2020-04-22, bjorg): since the message could not be sent to the dead-letter queue, // the next best action is to let Lambda retry it; unfortunately, there is no way // of knowing how many attempts have occurred already. // unable to forward message to dead-letter queue; report failure to lambda so it can retry metrics.Add(("MessageFailed.Count", 1, LambdaMetricUnit.Count)); throw; } return($"ERROR: {e.Message}".ToStream()); } } finally { _currentRecord = null; LogMetric(metrics); } }
private Exception Abort(AcknowledgeAction acknowledgeAction) => Abort(new APIGatewayProxyResponse { StatusCode = 200, Body = LambdaSerializer.Serialize(acknowledgeAction), Headers = new Dictionary <string, string> { ["ContentType"] = "application/json" } });
public override sealed async Task <BotResponse> ProcessMessageAsync(BotRequest request) { // check if there is a state object to load State = !string.IsNullOrEmpty(request.Bot?.InternalState) ? LambdaSerializer.Deserialize <TState>(request.Bot.InternalState) : new TState(); LogInfo($"Starting State:\n{LambdaSerializer.Serialize(State)}"); // dispatch to specific method based on request command BotResponse response; switch (request.Command) { case BotCommand.GetBuild: // bot configuration request response = new BotResponse { BotBuild = await GetBuildAsync() }; break; case BotCommand.GetAction: // bot action request try { // capture request fields for easy access GameSession = request.Session; Bot = request.Bot; GameClient = new LambdaRobotsGameClient(GameSession.ApiUrl, Bot.Id, HttpClient); // initialize a default empty action _action = new GetActionResponse(); // get bot action await GetActionAsync(); // generate response _action.BotState = LambdaSerializer.Serialize(State); response = new BotResponse { BotAction = _action, }; } finally { Bot = null; GameClient = null; } break; default: // unrecognized request throw new ApplicationException($"unexpected request: '{request.Command}'"); } // log response and return LogInfo($"Final State:\n{LambdaSerializer.Serialize(State)}"); return(response); }
//--- Methods --- /// <summary> /// The <see cref="ProcessMessageStreamAsync(Stream)"/> method is overridden to /// provide specific behavior for this base class. /// </summary> /// <remarks> /// This method cannot be overridden. /// </remarks> /// <param name="stream">The stream with the request payload.</param> /// <returns>The task object representing the asynchronous operation.</returns> public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream) { var schedule = LambdaSerializer.Deserialize <LambdaScheduleEvent>(stream); LogInfo($"received schedule event '{schedule.Name ?? schedule.Id}'"); await ProcessEventAsync(schedule); return("Ok".ToStream()); }
protected async Task <bool> Respond(SlackRequest request, SlackResponse response) { var httpResponse = await HttpClient.SendAsync(new HttpRequestMessage { RequestUri = new Uri(request.ResponseUrl), Method = HttpMethod.Post, Content = new StringContent(LambdaSerializer.Serialize(response)) }); return(httpResponse.StatusCode == HttpStatusCode.OK); }
public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream) { // sns event deserialization LogInfo("reading message stream"); SlackRequest request; try { request = LambdaSerializer.Deserialize <SlackRequest>(stream); } catch (Exception e) { LogError(e, "failed during Slack request deserialization"); return($"ERROR: {e.Message}".ToStream()); } // capture standard output and error output so we can send it to slack instead using (var consoleOutWriter = new StringWriter()) using (var consoleErrorWriter = new StringWriter()) { var consoleOutOriginal = Console.Out; var consoleErrorOriginal = Console.Error; try { // redirect the console output and error streams so we can emit them later to slack Console.SetOut(consoleOutWriter); Console.SetError(consoleErrorWriter); // validate the slack token (assuming one was configured) if (!(_slackVerificationToken?.Equals(request.Token) ?? true)) { throw new SlackVerificationTokenMismatchException(); } // handle slack request await ProcessSlackRequestAsync(request); } catch (Exception e) { LogError(e); Console.Error.WriteLine(e); } finally { Console.SetOut(consoleOutOriginal); Console.SetError(consoleErrorOriginal); } // send console output to slack as an in_channel response var output = consoleOutWriter.ToString(); if (output.Length > 0) { await RespondInChannel(request, output); } // send console error to slack as an ephemeral response (only visible to the requesting user) var error = consoleErrorWriter.ToString(); if (error.Length > 0) { await RespondEphemeral(request, error); } } return("Ok".ToStream()); }
//--- Methods --- /// <summary> /// The <see cref="ProcessMessageStreamAsync(Stream)"/> deserializes the request stream into /// a <typeparamref name="TRequest"/> instance and invokes the <see cref="ProcessMessageAsync(TRequest)"/> method. /// </summary> /// <remarks> /// This method is <c>sealed</c> and cannot be overridden. /// </remarks> /// <param name="stream">The stream with the request payload.</param> /// <returns>The task object representing the asynchronous operation.</returns> public override sealed async Task <Stream> ProcessMessageStreamAsync(Stream stream) { var request = LambdaSerializer.Deserialize <TRequest>(stream); var response = await ProcessMessageAsync(request); var responseStream = new MemoryStream(); LambdaSerializer.Serialize(response, responseStream); responseStream.Position = 0; return(responseStream); }
private Task NotifyAsync <T>(string userId, string channelId, T notification, int delay = 0) where T : ANotification => _sqsClient.SendMessageAsync(new Amazon.SQS.Model.SendMessageRequest { MessageBody = LambdaSerializer.Serialize(new BroadcastMessage { UserId = userId, ChannelId = channelId, Payload = LambdaSerializer.Serialize(notification) } ), QueueUrl = _notifyQueueUrl, DelaySeconds = delay });
private async Task WriteResponse( CloudFormationResourceRequest <TProperties> rawRequest, CloudFormationResourceResponse <TAttributes> rawResponse ) { Exception exception = null; var backoff = TimeSpan.FromMilliseconds(100); // write response to pre-signed S3 URL for (var i = 0; i < MAX_SEND_ATTEMPTS; ++i) { try { if (rawRequest.ResponseURL == null) { throw new InvalidOperationException("ResponseURL is missing"); } var httpResponse = await HttpClient.SendAsync(new HttpRequestMessage { RequestUri = new Uri(rawRequest.ResponseURL), Method = HttpMethod.Put, Content = new ByteArrayContent(Encoding.UTF8.GetBytes(LambdaSerializer.Serialize(rawResponse))) }); if (httpResponse.StatusCode != HttpStatusCode.OK) { throw new LambdaCustomResourceException( "PUT operation to pre-signed S3 URL failed with status code: {0} [{1} {2}] = {3}", httpResponse.StatusCode, rawRequest.RequestType, rawRequest.ResourceType ?? "<MISSING>", await httpResponse.Content.ReadAsStringAsync() ); } return; } catch (InvalidOperationException e) { exception = e; break; } catch (Exception e) { exception = e; LogErrorAsWarning(e, "writing response to pre-signed S3 URL failed"); await Task.Delay(backoff); backoff = TimeSpan.FromSeconds(backoff.TotalSeconds * 2); } } if (exception == null) { exception = new ShouldNeverHappenException($"ALambdaCustomResourceFunction.WriteResponse failed w/o an explicit"); } // max attempts have been reached; fail permanently and record the failed request for playback LogError(exception); await RecordFailedMessageAsync(LambdaLogLevel.ERROR, FailedMessageOrigin.CloudFormation, LambdaSerializer.Serialize(rawRequest), exception); }
public void ValidObjectShouldSerialize() { var serializer = new LambdaSerializer(); var stream = new MemoryStream(); var obj = new TestRequest() { Test = "Test" }; serializer.Serialize(obj, stream); stream.Position = 0; Assert.Equal("{\"Test\":\"Test\"}", new StreamReader(stream).ReadToEnd()); }
public void InvalidJsonShouldNotDeserialize() { var serializer = new LambdaSerializer(); var stream = new MemoryStream(); var json = "123"; var sw = new StreamWriter(stream); sw.Write(json); sw.Flush(); stream.Position = 0; Assert.Throws(typeof(LambdaException), () => serializer.Deserialize <TestRequest>(stream)); }
public override async Task ProcessMessageAsync(Message message) { await Task.WhenAll( _sqsClient.SendMessageAsync(_queueUrl, LambdaSerializer.Serialize(new Message { Counter = message.Counter + 1 })), _sqsClient.SendMessageAsync(_queueUrl, LambdaSerializer.Serialize(new Message { Counter = message.Counter + 2 })) ); throw new Exception("oops!"); }
public override async Task <FunctionResponse> ProcessMessageAsync(CloudWatchEvent <EventDetails> request) { LogInfo($"Version = {request.Version}"); LogInfo($"Account = {request.Account}"); LogInfo($"Region = {request.Region}"); LogInfo($"Detail = {LambdaSerializer.Serialize(request.Detail)}"); LogInfo($"DetailType = {request.DetailType}"); LogInfo($"Source = {request.Source}"); LogInfo($"Time = {request.Time}"); LogInfo($"Id = {request.Id}"); LogInfo($"Resources = [{string.Join(",", request.Resources ?? Enumerable.Empty<string>())}]"); LogInfo($"Latency = {DateTime.UtcNow - request.Time}"); return(new FunctionResponse()); }
public override async Task <MacroResponse> ProcessMessageAsync(MacroRequest request) { LogInfo($"AwsRegion = {request.Region}"); LogInfo($"AccountID = {request.AccountId}"); LogInfo($"Fragment = {LambdaSerializer.Serialize(request.Fragment)}"); LogInfo($"TransformID = {request.TransformId}"); LogInfo($"Params = {LambdaSerializer.Serialize(request.Params)}"); LogInfo($"RequestID = {request.RequestId}"); LogInfo($"TemplateParameterValues = {LambdaSerializer.Serialize(request.TemplateParameterValues)}"); // macro for string operations try { if (!request.Params.TryGetValue("Value", out var value)) { throw new ArgumentException("missing parameter: 'Value"); } if (!(value is string text)) { throw new ArgumentException("parameter 'Value' must be a string"); } string result; switch (request.TransformId) { case "StringToUpper": result = text.ToUpper(); break; case "StringToLower": result = text.ToLower(); break; default: throw new NotSupportedException($"requested operation is not supported: '{request.TransformId}'"); } // return successful response return(new MacroResponse { RequestId = request.RequestId, Status = "SUCCESS", Fragment = result }); } catch (Exception e) { // an error occurred return(new MacroResponse { RequestId = request.RequestId, Status = $"ERROR: {e.Message}" }); } }
// [Route("$connect")] public async Task <APIGatewayProxyResponse> OpenConnectionAsync(APIGatewayProxyRequest request) { LogInfo($"Connected: {request.RequestContext.ConnectionId}"); // verify client authorization string headerBase64Json = null; if (!(request.QueryStringParameters?.TryGetValue("header", out headerBase64Json) ?? false)) { // reject connection request return(new APIGatewayProxyResponse { StatusCode = (int)HttpStatusCode.Unauthorized }); } ConnectionHeader header; try { var headerJson = Encoding.UTF8.GetString(Convert.FromBase64String(headerBase64Json)); header = LambdaSerializer.Deserialize <ConnectionHeader>(headerJson); } catch { // reject connection request return(new APIGatewayProxyResponse { StatusCode = (int)HttpStatusCode.BadRequest }); } if ( (header.Host != request.RequestContext.DomainName) || (header.ApiKey != _clientApiKey) || !Guid.TryParse(header.Id, out _) ) { // reject connection request return(new APIGatewayProxyResponse { StatusCode = (int)HttpStatusCode.Forbidden }); } // create new connection record await _dataTable.CreateConnectionRecordAsync(new ConnectionRecord { ConnectionId = request.RequestContext.ConnectionId, State = ConnectionState.New, ApplicationId = header.Id, Bearer = request.RequestContext.Authorizer?.Claims }); return(new APIGatewayProxyResponse { StatusCode = (int)HttpStatusCode.OK }); }
public void ValidJsonShouldDeserialize() { var serializer = new LambdaSerializer(); var stream = new MemoryStream(); var json = "{\"Test\":\"Test\"}"; var sw = new StreamWriter(stream); sw.Write(json); sw.Flush(); stream.Position = 0; var d = serializer.Deserialize <TestRequest>(stream); Assert.Equal("Test", d.Test); }
public override async Task ProcessEventAsync(EventDetails details) { var request = CurrentEvent; LogInfo($"Version = {request.Version}"); LogInfo($"Account = {request.Account}"); LogInfo($"Region = {request.Region}"); LogInfo($"Detail = {LambdaSerializer.Serialize(details)}"); LogInfo($"DetailType = {request.DetailType}"); LogInfo($"Source = {request.Source}"); LogInfo($"Time = {request.Time}"); LogInfo($"Id = {request.Id}"); LogInfo($"Resources = [{string.Join(",", request.Resources ?? Enumerable.Empty<string>())}]"); LogInfo($"Latency = {DateTime.UtcNow - request.Time}"); }
//--- Methods --- /// <summary> /// The <see cref="InitializeEpilogueAsync()"/> method is invoked to complet the initialization of the /// Lambda function. This is the last of three methods that are invoked to initialize the Lambda function. /// </summary> /// <returns>The task object representing the asynchronous operation.</returns> protected override async Task InitializeEpilogueAsync() { await base.InitializeEpilogueAsync(); // NOTE (2019-04-15, bjorg): we initialize the invocation target directory after the function environment // initialization so that 'CreateInvocationTargetInstance()' can access the environment variables if need be. // read optional api-gateway-mappings file _directory = new ApiGatewayInvocationTargetDirectory(CreateInvocationTargetInstance, LambdaSerializer); if (File.Exists("api-mappings.json")) { var mappings = LambdaSerializer.Deserialize <ApiGatewayInvocationMappings>(File.ReadAllText("api-mappings.json")); if (mappings.Mappings == null) { throw new InvalidDataException("missing 'Mappings' property in 'api-mappings.json' file"); } foreach (var mapping in mappings.Mappings) { if (mapping.Method == null) { throw new InvalidDataException("missing 'Method' property in 'api-mappings.json' file"); } if (mapping.RestApi != null) { LogInfo($"Mapping REST API '{mapping.RestApi}' to {mapping.Method}"); _directory.Add(mapping.RestApi, mapping.Method); } else if (mapping.WebSocket != null) { LogInfo($"Mapping WebSocket route '{mapping.WebSocket}' to {mapping.Method}"); _directory.Add(mapping.WebSocket, mapping.Method); } else { throw new InvalidDataException("missing 'RestApi/WebSocket' property in 'api-mappings.json' file"); } } } }
public override async Task ProcessEventAsync(BitcoinPriceEvent message) { // TO-DO: add business logic LogInfo($"Received: {LambdaSerializer.Serialize(message)}"); }
public override async Task <KinesisFirehoseResponse> ProcessMessageAsync(KinesisFirehoseEvent request) { // NOTE (2018-12-11, bjorg): this function is responsible for error logs parsing; therefore, it CANNOT error out itself; // instead, it must rely on aggressive exception handling and redirect those message where appropriate. ResetMetrics(); var response = new KinesisFirehoseResponse { Records = new List <KinesisFirehoseResponse.FirehoseRecord>() }; _approximateResponseSize = 0; var reingestedCount = 0; try { for (var recordIndex = 0; recordIndex < request.Records.Count; ++recordIndex) { var record = request.Records[recordIndex]; try { var logEventsMessage = ConvertRecordToLogEventsMessage(record); // validate log event if ( (logEventsMessage.LogGroup is null) || (logEventsMessage.MessageType is null) || (logEventsMessage.LogEvents is null) ) { LogWarn("invalid log events message (record-id: {0})", record.RecordId); RecordFailed(record); continue; } // skip log event from own module if ((Info.FunctionName != null) && logEventsMessage.LogGroup.Contains(Info.FunctionName)) { LogInfo($"skipping log events message from own event log (record-id: {record.RecordId})"); RecordDropped(record); continue; } // skip control log event if (logEventsMessage.MessageType == "CONTROL_MESSAGE") { LogInfo($"skipping control log events message (record-id: {record.RecordId})"); RecordDropped(record); continue; } // check if this log event is expected if (logEventsMessage.MessageType != "DATA_MESSAGE") { LogWarn("unknown log events message type '{1}' (record-id: {0})", record.RecordId, logEventsMessage.MessageType); RecordFailed(record); continue; } // check if log group belongs to a Lambda function OwnerMetaData?owner; if (logEventsMessage.LogGroup.StartsWith(LAMBDA_LOG_GROUP_PREFIX, StringComparison.Ordinal)) { // use CloudWatch log name to identify owner of the log event var functionId = logEventsMessage.LogGroup.Substring(LAMBDA_LOG_GROUP_PREFIX.Length); owner = await GetOwnerMetaDataAsync($"F:{functionId}"); } else { owner = await GetOwnerMetaDataAsync($"L:{logEventsMessage.LogGroup}"); } // check if owner record exists if (owner == null) { LogInfo($"skipping log events message for unknown log-group (record-id: {record.RecordId}, log-group: {logEventsMessage.LogGroup})"); RecordDropped(record); continue; } // process entries in log event _convertedLogEvents.Clear(); var success = true; for (var logEventIndex = 0; logEventIndex < logEventsMessage.LogEvents.Count; ++logEventIndex) { var logEvent = logEventsMessage.LogEvents[logEventIndex]; try { await Logic.ProgressLogEntryAsync( owner, logEvent.Message, DateTimeOffset.FromUnixTimeMilliseconds(logEvent.Timestamp) ); } catch (Exception e) { if (owner.FunctionId != null) { LogError(e, "log event [{1}] processing failed (function-id: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, logEvent.Message ?? "<null>", owner.FunctionId); } else if (owner.AppId != null) { LogError(e, "log event [{1}] processing failed (app-id: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, logEvent.Message ?? "<null>", owner.AppId); } else { LogError(e, "log event [{1}] processing failed (log-group: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, logEvent.Message ?? "<null>", logEventsMessage.LogGroup); } // mark this log events message as failed and stop processing more log events success = false; break; } } // check outcome of processing log events message if (success) { // check if any log events were converted if (_convertedLogEvents.Any()) { // calculate size of the converted log events var convertedLogEventsSize = _convertedLogEvents.Sum(convertedLogEvent => convertedLogEvent.SerializedByteCount); if ((_approximateResponseSize + convertedLogEventsSize) > RESPONSE_SIZE_LIMIT) { // check if response size was exceeded on first record if (response.Records.Count == 0) { // skip record since it's the first record and we cannot serialize it due to response size limits LogWarn("record too large to convert (record-id: {0})", record.RecordId); RecordFailed(record); } else { LogInfo($"reached Lambda response limit (response: {_approximateResponseSize:N0}, limit: {RESPONSE_SIZE_LIMIT:N0})"); // reingest remaining records since the response will be too large otherwise var firehoseDeliveryStream = request.DeliveryStreamArn.Split('/').Last(); var reingestedRecords = request.Records.Skip(recordIndex).Select(record => new Record { Data = new MemoryStream(Convert.FromBase64String(record.Base64EncodedData)) }).ToList(); await FirehoseClient.PutRecordBatchAsync(firehoseDeliveryStream, reingestedRecords); reingestedCount += reingestedRecords.Count; // drop reingested records for (; recordIndex < request.Records.Count; ++recordIndex) { var droppedRecord = request.Records[recordIndex]; LogInfo($"reingested record (record-id: {droppedRecord.RecordId}"); RecordDropped(droppedRecord); } } } else { _approximateResponseSize += convertedLogEventsSize; // measure how long it took to successfully process the first CloudWatch Log event in the record if (logEventsMessage.LogEvents.Any()) { try { var timestamp = DateTimeOffset.FromUnixTimeMilliseconds(logEventsMessage.LogEvents.First().Timestamp); LogMetric("LogEvent.Latency", (DateTimeOffset.UtcNow - timestamp).TotalMilliseconds, LambdaMetricUnit.Milliseconds); } catch (Exception e) { LogError(e, "report log event latency failed"); } } // emit LambdaSharp events from converted log entries await ProcessConvertedLogEntriesAsync(); LogInfo($"finished converting log events (converted {_convertedLogEvents.Count:N0}, skipped {logEventsMessage.LogEvents.Count - _convertedLogEvents.Count:N0}, record-id: {record.RecordId})"); RecordSuccess(record, _convertedLogEvents.Aggregate("", (accumulator, convertedLogEvent) => accumulator + convertedLogEvent.Json + "\n")); } } else { LogInfo($"dropped record (record-id: {record.RecordId}"); RecordDropped(record); } } else { // nothing to log since error was already logged RecordFailed(record); } } catch (Exception e) { LogError(e, "record failed (record-id: {0})", record.RecordId); RecordFailed(record); } } // show processing outcome var okResponsesCount = response.Records.Count(r => r.Result == KinesisFirehoseResponse.TRANSFORMED_STATE_OK); var failedResponsesCount = response.Records.Count(r => r.Result == KinesisFirehoseResponse.TRANSFORMED_STATE_PROCESSINGFAILED); var droppedResponsesCount = response.Records.Count(r => r.Result == KinesisFirehoseResponse.TRANSFORMED_STATE_DROPPED) - reingestedCount; LogInfo($"processed {request.Records.Count:N0} records (success: {okResponsesCount}, failed: {failedResponsesCount:N0}, dropped: {droppedResponsesCount:N0}, reingested: {reingestedCount:N0})"); } finally { // NOTE (2020-04-21, bjorg): we don't expect this to fail; but since it's done at the end of the processing function, we // need to make sure it never fails; otherwise, the Kinesis stream processing is interrupted. try { ReportMetrics(); } catch (Exception e) { LogError(e, "report metrics failed"); } // send accumulated events try { PurgeEventEntries(); } catch (Exception exception) { // NOTE (2020-12-28, bjorg): don't use LogError() as it will eventually send an event, since there // is no other reporting mechanism for LambdaSharp.Core otherwise. Provider.Log($"EXCEPTION: {exception}\n"); } } return(response); // local functions LogEventsMessage ConvertRecordToLogEventsMessage(KinesisFirehoseEvent.FirehoseRecord record) { // deserialize kinesis record into a CloudWatch Log events message using var sourceStream = new MemoryStream(Convert.FromBase64String(record.Base64EncodedData)); using var recordData = new MemoryStream(); using (var gzip = new GZipStream(sourceStream, CompressionMode.Decompress)) { gzip.CopyTo(recordData); recordData.Position = 0; } return(LambdaSerializer.Deserialize <LogEventsMessage>(Encoding.UTF8.GetString(recordData.ToArray()))); } void RecordSuccess(KinesisFirehoseEvent.FirehoseRecord record, string data) => response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord { RecordId = record.RecordId, Result = KinesisFirehoseResponse.TRANSFORMED_STATE_OK, Base64EncodedData = Convert.ToBase64String(Encoding.UTF8.GetBytes(data)) }); void RecordFailed(KinesisFirehoseEvent.FirehoseRecord record) => response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord { RecordId = record.RecordId, Result = KinesisFirehoseResponse.TRANSFORMED_STATE_PROCESSINGFAILED }); void RecordDropped(KinesisFirehoseEvent.FirehoseRecord record) => response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord { RecordId = record.RecordId, Result = KinesisFirehoseResponse.TRANSFORMED_STATE_DROPPED }); }
public override async Task <APIGatewayHttpApiV2ProxyResponse> ProcessMessageAsync(APIGatewayHttpApiV2ProxyRequest request) { LogInfo($"Message received at {request.RequestContext.Http.Method}:{request.RawPath}?{request.RawQueryString}"); // validate invocation method if (request.RequestContext.Http.Method != "POST") { LogWarn("Unsupported request method {0}", request.RequestContext.Http.Method); return(BadRequestResponse()); } // validate request token if ( !request.QueryStringParameters.TryGetValue("token", out var token) || (token != _httpApiToken) ) { LogWarn("Missing or invalid request token"); return(BadRequestResponse()); } // validate request websocket if ( !request.QueryStringParameters.TryGetValue("ws", out var connectionId) || string.IsNullOrEmpty(connectionId) ) { LogWarn("Invalid websocket connection id"); return(BadRequestResponse()); } // validate request id if ( !request.QueryStringParameters.TryGetValue("rid", out var requestId) || string.IsNullOrEmpty(requestId) ) { LogWarn("Invalid request id"); return(BadRequestResponse()); } // check if request is a subscription confirmation var topicSubscription = LambdaSerializer.Deserialize <TopicSubscriptionPayload>(request.Body); if (topicSubscription.Type == "SubscriptionConfirmation") { return(await ConfirmSubscription(connectionId, requestId, topicSubscription)); } // validate SNS message var snsMessage = LambdaSerializer.Deserialize <SNSEvent.SNSMessage>(request.Body); if (snsMessage.Message == null) { LogWarn("Invalid SNS message received: {0}", request.Body); return(BadRequestResponse()); } // validate EventBridge event var eventBridgeEvent = LambdaSerializer.Deserialize <EventBridgeventPayload>(snsMessage.Message); if ( (eventBridgeEvent.Source == null) || (eventBridgeEvent.DetailType == null) || (eventBridgeEvent.Resources == null) ) { LogInfo("Invalid EventBridge event received: {0}", snsMessage.Message); return(BadRequestResponse()); } return(await DispatchEvent(connectionId, snsMessage.Message)); }
public override async Task <KinesisFirehoseResponse> ProcessMessageAsync(KinesisFirehoseEvent request) { // NOTE (2018-12-11, bjorg): this function is responsible for error logs parsing; therefore, it CANNOT error out itself; // instead, it must rely on aggressive exception handling and redirect those message where appropriate. ResetMetrics(); var response = new KinesisFirehoseResponse { Records = new List <KinesisFirehoseResponse.FirehoseRecord>() }; try { foreach (var record in request.Records) { try { // deserialize kinesis record into a CloudWatch Log event LogEventsMessage logEvent; using (var sourceStream = new MemoryStream(Convert.FromBase64String(record.Base64EncodedData))) using (var destinationStream = new MemoryStream()) { using (var gzip = new GZipStream(sourceStream, CompressionMode.Decompress)) { gzip.CopyTo(destinationStream); destinationStream.Position = 0; } logEvent = LambdaSerializer.Deserialize <LogEventsMessage>(Encoding.UTF8.GetString(destinationStream.ToArray())); } // validate log event if ( (logEvent.LogGroup == null) || (logEvent.MessageType == null) || (logEvent.LogEvents == null) ) { LogWarn("invalid record (record-id: {0})", record.RecordId); RecordFailed(record); continue; } // skip log event from own module if (logEvent.LogGroup.Contains(Info.FunctionName)) { LogInfo("skipping event from own event log (record-id: {0})", record.RecordId); RecordDropped(record); continue; } // skip control log event if (logEvent.MessageType == "CONTROL_MESSAGE") { LogInfo("skipping control message (record-id: {0})", record.RecordId); RecordDropped(record); continue; } // check if this log event is expected if (logEvent.MessageType != "DATA_MESSAGE") { LogWarn("unknown message type '{1}' (record-id: {0})", record.RecordId, logEvent.MessageType); RecordFailed(record); continue; } // check if log group belongs to a Lambda function OwnerMetaData?owner; if (logEvent.LogGroup.StartsWith(LAMBDA_LOG_GROUP_PREFIX, StringComparison.Ordinal)) { // use CloudWatch log name to identify owner of the log event var functionId = logEvent.LogGroup.Substring(LAMBDA_LOG_GROUP_PREFIX.Length); owner = await GetOwnerMetaDataAsync($"F:{functionId}"); } else { owner = await GetOwnerMetaDataAsync($"L:{logEvent.LogGroup}"); } // check if owner record exists if (owner == null) { LogInfo("skipping record for non-registered log-group (record-id: {0}, log-group: {1})", record.RecordId, logEvent.LogGroup); RecordDropped(record); continue; } // process entries in log event _convertedRecords.Clear(); var success = true; var logEventIndex = -1; foreach (var entry in logEvent.LogEvents) { ++logEventIndex; try { await Logic.ProgressLogEntryAsync( owner, entry.Message, DateTimeOffset.FromUnixTimeMilliseconds(entry.Timestamp) ); } catch (Exception e) { if (owner.FunctionId != null) { LogError(e, "log event entry [{1}] processing failed (function-id: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, entry.Message, owner.FunctionId); } else if (owner.AppId != null) { LogError(e, "log event entry [{1}] processing failed (app-id: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, entry.Message, owner.AppId); } else { LogError(e, "log event entry [{1}] processing failed (log-group: {3}, record-id: {0}):\n{2}", record.RecordId, logEventIndex, entry.Message, logEvent.LogGroup); } success = false; break; } } // record how long it took to process the CloudWatch Log event if (logEvent.LogEvents.Any()) { try { var timestamp = DateTimeOffset.FromUnixTimeMilliseconds(logEvent.LogEvents.First().Timestamp); LogMetric("LogEvent.Latency", (DateTimeOffset.UtcNow - timestamp).TotalMilliseconds, LambdaMetricUnit.Milliseconds); } catch (Exception e) { LogError(e, "report log event latency failed"); } } // record outcome if (success) { if (_convertedRecords.Any()) { LogInfo($"finished log events record (converted {_convertedRecords.Count:N0}, skipped {logEvent.LogEvents.Count - _convertedRecords.Count:N0}, record-id: {record.RecordId})"); RecordSuccess(record, _convertedRecords.Aggregate("", (accumulator, convertedRecord) => accumulator + convertedRecord + "\n")); } else { LogInfo($"dropped record (record-id: {record.RecordId}"); RecordDropped(record); } } else { // nothing to log since error was already logged RecordFailed(record); } } catch (Exception e) { LogError(e, "record failed (record-id: {0})", record.RecordId); RecordFailed(record); } } } finally { // NOTE (2020-04-21, bjorg): we don't expect this to fail; but since it's done at the end of the processing function, we // need to make sure it never fails; otherwise, the Kinesis stream processing is interrupted. try { ReportMetrics(); } catch (Exception e) { LogError(e, "report metrics failed"); } // send accumulated events try { PurgeEventEntries(); } catch (Exception exception) { Provider.Log($"EXCEPTION: {exception}\n"); } } return(response); // local functions void RecordSuccess(KinesisFirehoseEvent.FirehoseRecord record, string data) => response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord { RecordId = record.RecordId, Result = KinesisFirehoseResponse.TRANSFORMED_STATE_OK, Base64EncodedData = Convert.ToBase64String(Encoding.UTF8.GetBytes(data)) }); void RecordFailed(KinesisFirehoseEvent.FirehoseRecord record) => response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord { RecordId = record.RecordId, Result = KinesisFirehoseResponse.TRANSFORMED_STATE_PROCESSINGFAILED }); void RecordDropped(KinesisFirehoseEvent.FirehoseRecord record) => response.Records.Add(new KinesisFirehoseResponse.FirehoseRecord { RecordId = record.RecordId, Result = KinesisFirehoseResponse.TRANSFORMED_STATE_DROPPED }); }
//--- Methods --- /// <summary> /// The <see cref="Deserialize(string)"/> method converts the SNS topic message from string to a typed instance. /// </summary> /// <remarks> /// This method invokes <see cref="Amazon.Lambda.Core.ILambdaSerializer.Deserialize{TMessage}(Stream)"/> to convert the SNS topic message string /// into a <paramtyperef name="TMessage"/> instance. Override this method to provide a custom message deserialization implementation. /// </remarks> /// <param name="body">The SNS topic message.</param> /// <returns>The deserialized SNS topic message.</returns> public virtual TMessage Deserialize(string body) => LambdaSerializer.Deserialize <TMessage>(body);
public override async Task ProcessMessageAsync(SNSEvent.SNSMessage sns) { var evt = LambdaSerializer.Deserialize <Event>(sns.Message); LogInfo($"Received {evt.EventType}: {evt.Message}"); }
public override async Task <APIGatewayHttpApiV2ProxyResponse> ProcessMessageAsync(APIGatewayHttpApiV2ProxyRequest request) { LogInfo($"Message received at {request.RequestContext.Http.Method}:{request.RawPath}?{request.RawQueryString}"); // validate invocation method if (request.RequestContext.Http.Method != "POST") { LogInfo("Unsupported request method {0}", request.RequestContext.Http.Method); return(BadRequest()); } // validate request token if ( !request.QueryStringParameters.TryGetValue("token", out var token) || (token != _httpApiToken) ) { LogInfo("Missing or invalid request token"); return(BadRequest()); } // validate request websocket if ( !request.QueryStringParameters.TryGetValue("ws", out var connectionId) || string.IsNullOrEmpty(connectionId) ) { LogInfo("Invalid websocket connection id"); return(BadRequest()); } // validate request id if ( !request.QueryStringParameters.TryGetValue("rid", out var requestId) || string.IsNullOrEmpty(requestId) ) { LogInfo("Invalid request id"); return(BadRequest()); } // check if request is a subscription confirmation var topicSubscription = LambdaSerializer.Deserialize <TopicSubscriptionPayload>(request.Body); if (topicSubscription.Type == "SubscriptionConfirmation") { // confirm it's for the expected topic ARN if (topicSubscription.TopicArn != _eventTopicArn) { LogWarn("Wrong Topic ARN for subscription confirmation (Expected: {0}, Received: {1})", _eventTopicArn, topicSubscription.TopicArn); return(BadRequest()); } // confirm subscription await HttpClient.GetAsync(topicSubscription.SubscribeURL); // send welcome action to websocket connection await SendMessageToConnection(new AcknowledgeAction { RequestId = requestId, Status = "Ok" }, connectionId); return(Success("Confirmed")); } // validate SNS message var snsMessage = LambdaSerializer.Deserialize <SNSEvent.SNSMessage>(request.Body); if (snsMessage.Message == null) { LogWarn("Invalid SNS message received: {0}", request.Body); return(BadRequest()); } // validate CloudWatch event var cloudWatchEvent = LambdaSerializer.Deserialize <CloudWatchEventPayload>(snsMessage.Message); if ( (cloudWatchEvent.Source == null) || (cloudWatchEvent.DetailType == null) || (cloudWatchEvent.Resources == null) ) { LogInfo("Invalid CloudWatch event received: {0}", snsMessage.Message); return(BadRequest()); } // check if the keep-alive event was received if ( (cloudWatchEvent.Source == "aws.events") && (cloudWatchEvent.DetailType == "Scheduled Event") && (cloudWatchEvent.Resources.Count == 1) && (cloudWatchEvent.Resources[0] == _keepAliveRuleArn) ) { // send keep-alive action to websocket connection await SendMessageToConnection(new KeepAliveAction(), connectionId); return(Success("Ok")); } // determine what rules are matching JObject evt; try { evt = JObject.Parse(snsMessage.Message); } catch (Exception e) { LogError(e, "invalid message"); return(BadRequest()); } var rules = await _dataTable.GetConnectionRulesAsync(connectionId); var matchedRules = rules .Where(rule => { try { var pattern = JObject.Parse(rule.Pattern); return(EventPatternMatcher.IsMatch(evt, pattern)); } catch (Exception e) { LogError(e, "invalid event pattern: {0}", rule.Pattern); return(false); } }).Select(rule => rule.Rule) .ToList(); if (matchedRules.Any()) { await SendMessageToConnection( new EventAction { Rules = matchedRules, Source = cloudWatchEvent.Source, Type = cloudWatchEvent.DetailType, Event = snsMessage.Message }, connectionId ); } return(Success("Ok")); // local functions APIGatewayHttpApiV2ProxyResponse Success(string message) => new APIGatewayHttpApiV2ProxyResponse { Body = message, Headers = new Dictionary <string, string> { ["Content-Type"] = "text/plain" }, StatusCode = (int)HttpStatusCode.OK }; APIGatewayHttpApiV2ProxyResponse BadRequest() => new APIGatewayHttpApiV2ProxyResponse { Body = "Bad Request", Headers = new Dictionary <string, string> { ["Content-Type"] = "text/plain" }, StatusCode = (int)HttpStatusCode.BadRequest }; }
private async Task <APIGatewayHttpApiV2ProxyResponse> DispatchEvent(string connectionId, string message) { // validate EventBridge event var eventBridgeEvent = LambdaSerializer.Deserialize <EventBridgeventPayload>(message); if ( (eventBridgeEvent.Source == null) || (eventBridgeEvent.DetailType == null) || (eventBridgeEvent.Resources == null) ) { LogInfo("Invalid EventBridge event received: {0}", message); return(BadRequestResponse()); } // check if the keep-alive event was received if ( (eventBridgeEvent.Source == "aws.events") && (eventBridgeEvent.DetailType == "Scheduled Event") && (eventBridgeEvent.Resources?.Count == 1) && (eventBridgeEvent.Resources[0] == _keepAliveRuleArn) ) { // retrieve connection record var connection = await _dataTable.GetConnectionRecordAsync(connectionId); if (connection == null) { return(SuccessResponse("Gone")); } if (connection.State != ConnectionState.Open) { return(SuccessResponse("Ignored")); } // send keep-alive action to websocket connection LogInfo("KeepAlive tick"); await SendActionToConnection(new KeepAliveAction(), connectionId); return(SuccessResponse("Ok")); } // determine what rules are matching the event JObject evt; try { evt = JObject.Parse(message); } catch (Exception e) { LogError(e, "invalid message"); return(BadRequestResponse()); } var rules = await _dataTable.GetAllRuleRecordAsync(connectionId); var matchedRules = rules .Where(rule => { try { var pattern = JObject.Parse(rule.Pattern); return(EventPatternMatcher.IsMatch(evt, pattern)); } catch (Exception e) { LogError(e, "invalid event pattern: {0}", rule.Pattern); return(false); } }).Select(rule => rule.Rule) .ToList(); if (matchedRules.Any()) { LogInfo($"Event matched {matchedRules.Count():N0} rules: {string.Join(", ", matchedRules)}"); await SendActionToConnection( new EventAction { Rules = matchedRules, Source = eventBridgeEvent.Source, Type = eventBridgeEvent.DetailType, Event = message }, connectionId ); } else { LogInfo("Event matched no rules"); } return(SuccessResponse("Ok")); }