private async Task ProcessEventRecord(KinesisEvent.KinesisEventRecord record, bool writeEventsToQueue)
{
_logger.Trace("Beginning ProcessEventRecord");
_logger.Debug($"Kinesis EventId:{record.EventId} EventName: {record.EventName} EventSource: {record.EventSource} EventVersion: {record.EventVersion} EventSourceARN {record.EventSourceARN} InvokeIdentityARN: {record.InvokeIdentityArn} Kinesis Time: {record.Kinesis.ApproximateArrivalTimestamp} Kinesis ParitionKey: {record.Kinesis.PartitionKey} Kinesis SeqNum: {record.Kinesis.SequenceNumber}");
string recordData = GetRecordContents(record.Kinesis);
_logger.Debug($"RecordData:{recordData}");
//FOR CREATING TEST DATA
//var plainTextBytes = System.Text.Encoding.UTF8.GetBytes(recordData);
//string convertedRecordData = System.Convert.ToBase64String(plainTextBytes);
//Logger.Info(convertedRecordData);
if (!String.IsNullOrEmpty(recordData))
{
ConnectKinesisEventRecord streamRecord = ConvertRecordData(recordData);
if (streamRecord.LastEventType != HEARTBEAT_EVENTTYPE)
{
await ProcessEventToTable(streamRecord);
}
if (writeEventsToQueue)
{
await ProcessEventToQueue(recordData);
}
}
_logger.Trace("Ending ProcessEventRecord");
}
protected async Task <TBody> ParseRecord(KinesisEvent.KinesisEventRecord record)
{
if (record == null)
{
throw new ArgumentNullException(nameof(record));
}
if (record.Kinesis.Data is null)
{
throw new CriticalException($"Kinesis Data for EventId {record.EventId} is null");
}
Logger.Info($"Processing Kinesis Event message ID {record.EventId}.");
using var activity = EventProcessorActivitySource.StartActivity(nameof(ParseRecord));
activity?.SetTag("Kinesis.EventId", record.EventId);
activity?.SetTag("Kinesis.EventName", record.EventName);
activity?.SetTag("Kinesis.EventVersion", record.EventVersion);
activity?.SetTag("Kinesis.EventSource", record.EventSource);
activity?.SetTag("Kinesis.PartitionKey", record.Kinesis?.PartitionKey);
activity?.SetTag("Kinesis.ApproximateArrivalTimestamp", record.Kinesis?.ApproximateArrivalTimestamp);
Logger.Info("Reading Stream Content.");
using var reader = new StreamReader(record.Kinesis.Data, Encoding.UTF8);
var content = await reader.ReadToEndAsync().ConfigureAwait(false);
Logger.Info("Stream content read finished.");
Logger.Info("Deserializing Body Message.");
var body = DeserializeBody(content, record.Kinesis.PartitionKey);
Logger.Info("Body message deserialized.");
if (body != null)
{
body.EventId = record.EventId;
body.ApproximateArrivalTimestamp = record.Kinesis.ApproximateArrivalTimestamp;
body.Partition ??= record.Kinesis.PartitionKey;
body.TraceId ??= activity?.Id;
}
if (body?.TraceId != null && activity?.ParentId is null)
{
activity?.SetParentId(body.TraceId);
}
return(body);
}
protected async Task <T> ParseRecord <T>(KinesisEvent.KinesisEventRecord record) where T : IDataStream
{
if (record == null)
{
throw new ArgumentNullException(nameof(record));
}
Logger.Info($"Processing Kinesis Event message ID {record.EventId}.");
using var activity = EventProcessorActivitySource.StartActivity(nameof(ParseRecord));
activity?.SetTag("EventId", record.EventId);
activity?.SetTag("EventName", record.EventName);
activity?.SetTag("EventVersion", record.EventVersion);
activity?.SetTag("EventSource", record.EventSource);
try
{
Logger.Info("Reading Stream Content.");
using var reader = new StreamReader(record.Kinesis.Data, Encoding.UTF8);
var content = await reader.ReadToEndAsync().ConfigureAwait(false);
Logger.Info("Stream Content Read.");
Logger.Info("Creating DataStream Message.");
var body = DeserializeBody <T>(content, record.Kinesis.PartitionKey);
if (body != null)
{
body.EventId = record.EventId;
body.ApproximateArrivalTimestamp = record.Kinesis.ApproximateArrivalTimestamp;
}
Logger.Info("Invoking ProcessMessage.");
return(body);
}
catch (Exception ex)
{
Logger.Error(ex,
"Error Processing Message from Kinesis Event. Developer, you should take care of it!. Message: Id={EventId}, PartitionKey= {PartitionKey}",
record.EventId, record.Kinesis.PartitionKey);
throw;
}
}
private async void ProcessRecord(StreamRequestContext context, KinesisEvent.KinesisEventRecord record)
{
var correlationId = string.Empty;
try
{
var customerNotifications = StreamUtils.GetStreamRecord <CustomerNotificationDTO>(record.Kinesis.Data);
_logger.LogInfo($"Call Type: {customerNotifications.Event.RequestType} number: {customerNotifications.Customer.NotificationNumber} status:{customerNotifications.Event.RequestStatus}", context);
if (customerNotifications.Event.RequestType == "Digital" && customerNotifications.Event.RequestStatus == "REQUEST CONFIRMED")
{
HttpClient client = new HttpClient();
var caseId = customerNotifications.CaseData != null?customerNotifications.CaseData.CaseId.ToString() : "";
var clientId = customerNotifications.CaseData != null?customerNotifications.CaseData.ClientId.ToString() : "";
var callbackRefNumber = customerNotifications.Customer != null?customerNotifications.Customer.CallbackRefNumber.ToString() : "";
var notificationNumber = customerNotifications.Customer != null?customerNotifications.Customer.NotificationNumber.ToString() : "";
var values = new Dictionary <string, string>
{
{ "CallbackRefNumber", callbackRefNumber }
};
var content = new FormUrlEncodedContent(values);
var response = await client.PostAsync("", content);
var responseString = await response.Content.ReadAsStringAsync();
}
}
catch (BatchProcessingException processingException)
{
throw;
}
catch (Exception ex)
{
throw;
}
}
private async Task ProcessEventRecord(KinesisEvent.KinesisEventRecord record, bool writeEventsToQueue)
{
_logger.Trace("Beginning ProcessEventRecord");
_logger.Debug($"Kinesis EventId:{record.EventId} EventName: {record.EventName} EventSource: {record.EventSource} EventVersion: {record.EventVersion} EventSourceARN {record.EventSourceARN} InvokeIdentityARN: {record.InvokeIdentityArn} Kinesis Time: {record.Kinesis.ApproximateArrivalTimestamp} Kinesis ParitionKey: {record.Kinesis.PartitionKey} Kinesis SeqNum: {record.Kinesis.SequenceNumber}");
string recordData = GetRecordContents(record.Kinesis);
_logger.Debug($"RecordData:{recordData}");
//FOR CREATING TEST DATA
//var plainTextBytes = System.Text.Encoding.UTF8.GetBytes(recordData);
//string convertedRecordData = System.Convert.ToBase64String(plainTextBytes);
//Logger.Info(convertedRecordData);
if (!String.IsNullOrEmpty(recordData))
{
var minimizedJSON = ShrinkEvent(recordData);
var eventRecord = ParseEvent(minimizedJSON);
if (eventRecord != null && MatchFilter(eventRecord))
{
_logger.Debug("Event matches filter");
ConnectKinesisEventRecord streamRecord = ConvertRecordData(minimizedJSON, eventRecord);
// if we were able to parse this event, send to DynamoDB and SQS.
if (streamRecord != null)
{
if (streamRecord.LastEventType != HEARTBEAT_EVENTTYPE)
{
await ProcessEventToTable(streamRecord);
}
if (writeEventsToQueue)
{
await ProcessEventToQueue(minimizedJSON, streamRecord.AgentARN);
}
}
}
}
_logger.Trace("Ending ProcessEventRecord");
}
public void Setup()
{
// APIGatewayProxy
_baseAPIGatewayProxyRequest = new APIGatewayProxyRequest
{
HttpMethod = "POST",
Path = "/test/path",
};
_baseAPIGatewayProxyResponse = new APIGatewayProxyResponse
{
StatusCode = (int)HttpStatusCode.OK,
};
// ApplicationLoadBalancer
var albRequestContext = new ApplicationLoadBalancerRequest.ALBRequestContext
{
Elb = new ApplicationLoadBalancerRequest.ElbInfo()
};
albRequestContext.Elb.TargetGroupArn = TestArn;
_baseApplicationLoadBalancerRequest = new ApplicationLoadBalancerRequest
{
HttpMethod = "POST",
Path = "/test/path",
RequestContext = albRequestContext,
};
_baseApplicationLoadBalancerResponse = new ApplicationLoadBalancerResponse
{
StatusCode = (int)HttpStatusCode.OK,
};
// SQSEvent
var sqsRecord = new SQSEvent.SQSMessage
{
EventSourceArn = TestArn
};
_baseSQSEvent = new SQSEvent
{
Records = new List <SQSEvent.SQSMessage> {
sqsRecord
},
};
// SNSEvent
var snsMessaage = new SNSEvent.SNSMessage()
{
Message = "Test Message",
};
var snsRecord = new SNSEvent.SNSRecord
{
EventSubscriptionArn = TestArn,
Sns = snsMessaage
};
_baseSNSEvent = new SNSEvent
{
Records = new List <SNSEvent.SNSRecord> {
snsRecord
},
};
// KinesisEvent
var kinesisRecord = new KinesisEvent.KinesisEventRecord
{
EventSourceARN = TestArn
};
_baseKinesisEvent = new KinesisEvent
{
Records = new List <KinesisEvent.KinesisEventRecord> {
kinesisRecord
},
};
// S3Event
var s3Record = new Amazon.S3.Util.S3EventNotification.S3EventNotificationRecord
{
S3 = new Amazon.S3.Util.S3EventNotification.S3Entity
{
Bucket = new Amazon.S3.Util.S3EventNotification.S3BucketEntity
{
Arn = TestArn
}
}
};
_baseS3Event = new S3Event
{
Records = new List <Amazon.S3.Util.S3EventNotification.S3EventNotificationRecord> {
s3Record
},
};
// DynamoDBEvent
var dynamoDBRecord = new DynamoDBEvent.DynamodbStreamRecord
{
EventSourceArn = TestArn
};
_baseDynamoDBEvent = new DynamoDBEvent
{
Records = new List <DynamoDBEvent.DynamodbStreamRecord> {
dynamoDBRecord
},
};
// KinesisFirehoseEvent
_baseKinesisFirehoseEvent = new KinesisFirehoseEvent
{
DeliveryStreamArn = TestArn,
};
}