/// <summary>
/// Parse the JSON string into a S3EventNotification object.
/// <para>
/// The function will try its best to parse input JSON string as best as it can.
/// It will not fail even if the JSON string contains unknown properties.
/// </para>
/// <exception cref="Amazon.Runtime.AmazonClientException">For any parsing errors</exception>
/// </summary>
public static S3EventNotification ParseJson(string json)
{
try
{
var data = JsonMapper.ToObject(json);
var s3Event = new S3EventNotification { Records = new List<S3EventNotificationRecord>() };
if (data["Records"] != null)
{
foreach (JsonData jsonRecord in data["Records"])
{
var record = new S3EventNotificationRecord();
record.EventVersion = GetValueAsString(jsonRecord, "eventVersion");
record.EventSource = GetValueAsString(jsonRecord, "eventSource");
record.AwsRegion = GetValueAsString(jsonRecord, "awsRegion");
record.EventVersion = GetValueAsString(jsonRecord, "eventVersion");
if (jsonRecord["eventTime"] != null)
record.EventTime = DateTime.Parse((string)jsonRecord["eventTime"], CultureInfo.InvariantCulture);
if (jsonRecord["eventName"] != null)
{
var eventName = (string)jsonRecord["eventName"];
if (!eventName.StartsWith("s3:", StringComparison.OrdinalIgnoreCase))
eventName = "s3:" + eventName;
record.EventName = EventType.FindValue(eventName);
}
if (jsonRecord["userIdentity"] != null)
{
var jsonUserIdentity = jsonRecord["userIdentity"];
record.UserIdentity = new UserIdentityEntity();
record.UserIdentity.PrincipalId = GetValueAsString(jsonUserIdentity, "principalId");
}
if (jsonRecord["requestParameters"] != null)
{
var jsonRequestParameters = jsonRecord["requestParameters"];
record.RequestParameters = new RequestParametersEntity();
record.RequestParameters.SourceIPAddress = GetValueAsString(jsonRequestParameters, "sourceIPAddress");
}
if (jsonRecord["responseElements"] != null)
{
var jsonResponseElements = jsonRecord["responseElements"];
record.ResponseElements = new ResponseElementsEntity();
record.ResponseElements.XAmzRequestId = GetValueAsString(jsonResponseElements, "x-amz-request-id");
record.ResponseElements.XAmzId2 = GetValueAsString(jsonResponseElements, "x-amz-id-2");
}
if (jsonRecord["s3"] != null)
{
var jsonS3 = jsonRecord["s3"];
record.S3 = new S3Entity();
record.S3.S3SchemaVersion = GetValueAsString(jsonS3, "s3SchemaVersion");
record.S3.ConfigurationId = GetValueAsString(jsonS3, "configurationId");
if (jsonS3["bucket"] != null)
{
var jsonBucket = jsonS3["bucket"];
record.S3.Bucket = new S3BucketEntity();
record.S3.Bucket.Name = GetValueAsString(jsonBucket, "name");
record.S3.Bucket.Arn = GetValueAsString(jsonBucket, "arn");
if (jsonBucket["ownerIdentity"] != null)
{
var jsonOwnerIdentity = jsonBucket["ownerIdentity"];
record.S3.Bucket.OwnerIdentity = new UserIdentityEntity();
record.S3.Bucket.OwnerIdentity.PrincipalId = GetValueAsString(jsonOwnerIdentity, "principalId");
}
}
if (jsonS3["object"] != null)
{
var jsonObject = jsonS3["object"];
record.S3.Object = new S3ObjectEntity();
record.S3.Object.Key = GetValueAsString(jsonObject, "key");
record.S3.Object.Size = GetValueAsLong(jsonObject, "size");
record.S3.Object.ETag = GetValueAsString(jsonObject, "eTag");
record.S3.Object.VersionId = GetValueAsString(jsonObject, "versionId");
}
}
s3Event.Records.Add(record);
}
}
return s3Event;
}
catch(Exception e)
{
throw new AmazonClientException("Failed to parse json string: " + e.Message, e);
}
}
/// <summary>
/// Parse the JSON string into a S3EventNotification object.
/// <para>
/// The function will try its best to parse input JSON string as best as it can.
/// It will not fail even if the JSON string contains unknown properties.
/// </para>
/// <exception cref="Amazon.Runtime.AmazonClientException">For any parsing errors</exception>
/// </summary>
public static S3EventNotification ParseJson(string json)
{
try
{
var data = JsonMapper.ToObject(json);
var s3Event = new S3EventNotification {
Records = new List <S3EventNotificationRecord>()
};
if (data["Records"] != null)
{
foreach (JsonData jsonRecord in data["Records"])
{
var record = new S3EventNotificationRecord();
record.EventVersion = GetValueAsString(jsonRecord, "eventVersion");
record.EventSource = GetValueAsString(jsonRecord, "eventSource");
record.AwsRegion = GetValueAsString(jsonRecord, "awsRegion");
record.EventVersion = GetValueAsString(jsonRecord, "eventVersion");
if (jsonRecord["eventTime"] != null)
{
record.EventTime = DateTime.Parse((string)jsonRecord["eventTime"], CultureInfo.InvariantCulture);
}
if (jsonRecord["eventName"] != null)
{
var eventName = (string)jsonRecord["eventName"];
if (!eventName.StartsWith("s3:", StringComparison.OrdinalIgnoreCase))
{
eventName = "s3:" + eventName;
}
record.EventName = EventType.FindValue(eventName);
}
if (jsonRecord["userIdentity"] != null)
{
var jsonUserIdentity = jsonRecord["userIdentity"];
record.UserIdentity = new UserIdentityEntity();
record.UserIdentity.PrincipalId = GetValueAsString(jsonUserIdentity, "principalId");
}
if (jsonRecord["requestParameters"] != null)
{
var jsonRequestParameters = jsonRecord["requestParameters"];
record.RequestParameters = new RequestParametersEntity();
record.RequestParameters.SourceIPAddress = GetValueAsString(jsonRequestParameters, "sourceIPAddress");
}
if (jsonRecord["responseElements"] != null)
{
var jsonResponseElements = jsonRecord["responseElements"];
record.ResponseElements = new ResponseElementsEntity();
record.ResponseElements.XAmzRequestId = GetValueAsString(jsonResponseElements, "x-amz-request-id");
record.ResponseElements.XAmzId2 = GetValueAsString(jsonResponseElements, "x-amz-id-2");
}
if (jsonRecord["s3"] != null)
{
var jsonS3 = jsonRecord["s3"];
record.S3 = new S3Entity();
record.S3.S3SchemaVersion = GetValueAsString(jsonS3, "s3SchemaVersion");
record.S3.ConfigurationId = GetValueAsString(jsonS3, "configurationId");
if (jsonS3["bucket"] != null)
{
var jsonBucket = jsonS3["bucket"];
record.S3.Bucket = new S3BucketEntity();
record.S3.Bucket.Name = GetValueAsString(jsonBucket, "name");
record.S3.Bucket.Arn = GetValueAsString(jsonBucket, "arn");
if (jsonBucket["ownerIdentity"] != null)
{
var jsonOwnerIdentity = jsonBucket["ownerIdentity"];
record.S3.Bucket.OwnerIdentity = new UserIdentityEntity();
record.S3.Bucket.OwnerIdentity.PrincipalId = GetValueAsString(jsonOwnerIdentity, "principalId");
}
}
if (jsonS3["object"] != null)
{
var jsonObject = jsonS3["object"];
record.S3.Object = new S3ObjectEntity();
record.S3.Object.Key = GetValueAsString(jsonObject, "key");
record.S3.Object.Size = GetValueAsLong(jsonObject, "size");
record.S3.Object.ETag = GetValueAsString(jsonObject, "eTag");
record.S3.Object.VersionId = GetValueAsString(jsonObject, "versionId");
}
}
s3Event.Records.Add(record);
}
}
return(s3Event);
}
catch (Exception e)
{
throw new AmazonClientException("Failed to parse json string: " + e.Message, e);
}
}
public static S3EventNotification ParseJson(string json)
{
//IL_0044: Unknown result type (might be due to invalid IL or missing references)
//IL_004a: Expected O, but got Unknown
//IL_03e6: Unknown result type (might be due to invalid IL or missing references)
try
{
JsonData val = JsonMapper.ToObject(json);
S3EventNotification s3EventNotification = new S3EventNotification
{
Records = new List <S3EventNotificationRecord>()
};
if (val.get_Item("Records") != null)
{
foreach (JsonData item in (IEnumerable)val.get_Item("Records"))
{
JsonData val2 = item;
S3EventNotificationRecord s3EventNotificationRecord = new S3EventNotificationRecord();
s3EventNotificationRecord.EventVersion = GetValueAsString(val2, "eventVersion");
s3EventNotificationRecord.EventSource = GetValueAsString(val2, "eventSource");
s3EventNotificationRecord.AwsRegion = GetValueAsString(val2, "awsRegion");
s3EventNotificationRecord.EventVersion = GetValueAsString(val2, "eventVersion");
if (val2.get_Item("eventTime") != null)
{
s3EventNotificationRecord.EventTime = DateTime.Parse((string)val2.get_Item("eventTime"), CultureInfo.InvariantCulture);
}
if (val2.get_Item("eventName") != null)
{
string text = (string)val2.get_Item("eventName");
if (!text.StartsWith("s3:", StringComparison.OrdinalIgnoreCase))
{
text = "s3:" + text;
}
s3EventNotificationRecord.EventName = EventType.FindValue(text);
}
if (val2.get_Item("userIdentity") != null)
{
JsonData data = val2.get_Item("userIdentity");
s3EventNotificationRecord.UserIdentity = new UserIdentityEntity();
s3EventNotificationRecord.UserIdentity.PrincipalId = GetValueAsString(data, "principalId");
}
if (val2.get_Item("requestParameters") != null)
{
JsonData data2 = val2.get_Item("requestParameters");
s3EventNotificationRecord.RequestParameters = new RequestParametersEntity();
s3EventNotificationRecord.RequestParameters.SourceIPAddress = GetValueAsString(data2, "sourceIPAddress");
}
if (val2.get_Item("responseElements") != null)
{
JsonData data3 = val2.get_Item("responseElements");
s3EventNotificationRecord.ResponseElements = new ResponseElementsEntity();
s3EventNotificationRecord.ResponseElements.XAmzRequestId = GetValueAsString(data3, "x-amz-request-id");
s3EventNotificationRecord.ResponseElements.XAmzId2 = GetValueAsString(data3, "x-amz-id-2");
}
if (val2.get_Item("s3") != null)
{
JsonData val3 = val2.get_Item("s3");
s3EventNotificationRecord.S3 = new S3Entity();
s3EventNotificationRecord.S3.S3SchemaVersion = GetValueAsString(val3, "s3SchemaVersion");
s3EventNotificationRecord.S3.ConfigurationId = GetValueAsString(val3, "configurationId");
if (val3.get_Item("bucket") != null)
{
JsonData val4 = val3.get_Item("bucket");
s3EventNotificationRecord.S3.Bucket = new S3BucketEntity();
s3EventNotificationRecord.S3.Bucket.Name = GetValueAsString(val4, "name");
s3EventNotificationRecord.S3.Bucket.Arn = GetValueAsString(val4, "arn");
if (val4.get_Item("ownerIdentity") != null)
{
JsonData data4 = val4.get_Item("ownerIdentity");
s3EventNotificationRecord.S3.Bucket.OwnerIdentity = new UserIdentityEntity();
s3EventNotificationRecord.S3.Bucket.OwnerIdentity.PrincipalId = GetValueAsString(data4, "principalId");
}
}
if (val3.get_Item("object") != null)
{
JsonData data5 = val3.get_Item("object");
s3EventNotificationRecord.S3.Object = new S3ObjectEntity();
s3EventNotificationRecord.S3.Object.Key = GetValueAsString(data5, "key");
s3EventNotificationRecord.S3.Object.Size = GetValueAsLong(data5, "size");
s3EventNotificationRecord.S3.Object.ETag = GetValueAsString(data5, "eTag");
s3EventNotificationRecord.S3.Object.VersionId = GetValueAsString(data5, "versionId");
}
}
s3EventNotification.Records.Add(s3EventNotificationRecord);
}
}
return(s3EventNotification);
}
catch (Exception ex)
{
throw new AmazonClientException("Failed to parse json string: " + ex.Message, ex);
}
}
/// <summary>
/// Processes a single manifest file and all of the report keys it contains
/// </summary>
/// <param name="item"></param>
/// <param name="context"></param>
/// <returns></returns>
private static async Task <Manifest> ProcessItemAsync(S3EventNotificationRecord item, string destinationBucket, ILambdaContext context)
{
context.LogInfo(JsonConvert.SerializeObject(item));
// Make sure the event was when a new object was created
if (item.EventName != EventType.ObjectCreatedPut && item.EventName != EventType.ObjectCreatedPost)
{
string Message = $"This Lambda function was triggered by a non ObjectCreated Put or Post event, {item.EventName}, for object {item.S3.Object.Key}; check the CloudFormation template configuration and S3 Event setup.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(null);
}
// Get the manifest file contents
GetObjectRequest Request = new GetObjectRequest()
{
BucketName = item.S3.Bucket.Name,
Key = item.S3.Object.Key
};
string Body = "";
using (GetObjectResponse Response = await _S3Client.GetObjectAsync(Request))
{
using (Stream ResponseStream = Response.ResponseStream)
{
using (StreamReader Reader = new StreamReader(ResponseStream))
{
Body = await Reader.ReadToEndAsync();
}
}
}
Manifest ManifestFile = Manifest.Build(Body);
string Prefix = GetDestinationPrefix(ManifestFile);
// Build the destination key map to link source key to destination key
Dictionary <string, string> DestinationKeyMap = ManifestFile.ReportKeys.ToDictionary(x => x, x => $"{Prefix}/{Path.GetFileName(x)}");
// If there are no destination keys
// then there is nothing to do, return
if (!DestinationKeyMap.Any())
{
string Message = $"No destination keys produced for s3://{Request.BucketName}/{Request.Key}";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(null);
}
// Copy all of the files over first to replace existing files, this way there
// is no period of time where a file may not exist and break an active query
List <Task <CopyResponse> > CopyTasks = new List <Task <CopyResponse> >();
// Initiate a copy object task for each key
foreach (KeyValuePair <string, string> KeySet in DestinationKeyMap)
{
try
{
context.LogInfo($"Copying CUR from s3://{item.S3.Bucket.Name}/{KeySet.Key} to s3://{_DestinationBucket}/{KeySet.Value}");
CopyTasks.Add(CopyObjectAsync(KeySet.Key, KeySet.Value, item.S3.Bucket.Name, _DestinationBucket));
}
catch (Exception e)
{
string Message = $"Failed to add a copy object task to the queue for s3://{item.S3.Bucket.Name}/{KeySet.Key} to s3://{_DestinationBucket}/{KeySet.Value}.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
}
// Process the copy object results
foreach (Task <CopyResponse> Response in CopyTasks.Interleaved())
{
try
{
CopyResponse Result = await Response;
if (Result.IsError)
{
string Message = $"Failed to copy s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey}.";
context.LogError(Message, Result.Exception);
await SNSNotify(Message, context);
return(null);
}
else
{
if (Result.Response.HttpStatusCode != HttpStatusCode.OK)
{
string Message = $"Failed to copy s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey} with http code {(int)Result.Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(null);
}
else
{
context.LogInfo($"Successfully copied CUR from s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey}.");
}
}
}
catch (Exception e)
{
string Message = $"Internal error processing the copy async task.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
}
// Delete all of the keys in the that are not the files we just copied over
List <KeyVersion> KeysToDelete;
try
{
// Find all keys under the same prefix, and that aren't one of the keys of the files that have been copied
KeysToDelete = await ListAllObjectsAsync(destinationBucket, Prefix, x => x.Where(y => !DestinationKeyMap.Values.Contains(y.Key)));
}
catch (Exception e)
{
context.LogError(e);
await SNSNotify($"{e.Message}\n{e.StackTrace}", context);
return(null);
}
// Delete the old CUR files in the destination bucket
try
{
if (KeysToDelete != null && KeysToDelete.Any())
{
int DeletedCount = await DeleteObjectsAsync(KeysToDelete, destinationBucket);
if (DeletedCount != KeysToDelete.Count)
{
string Message = $"Unable to delete all objects, expected to delete {KeysToDelete.Count} but only deleted {DeletedCount}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(null);
}
else
{
context.LogInfo($"Successfully deleted {DeletedCount} objects.");
}
}
}
catch (Exception e)
{
string Message = "Unable to delete all old CUR files.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
return(ManifestFile);
}
