private Manifest GetManifest()
{
string manifestFilePath = this.GetManifestFilePath();
Logger.Info("Downloading " + this.m_ManifestURL + " to " + manifestFilePath);
bool downloaded = false;
Exception capturedException = (Exception)null;
SplitDownloader.DownloadFile(this.m_ManifestURL, manifestFilePath, this.m_UserAgent, (SplitDownloader.DownloadFileProgressCb)((downloadedSize, totalSize) => Logger.Info("Downloaded (" + downloadedSize.ToString() + " bytes) out of " + totalSize.ToString())), (SplitDownloader.DownloadFileCompletedCb)(filePath =>
{
downloaded = true;
Logger.Info("Downloaded " + this.m_ManifestURL + " to " + filePath);
}), (SplitDownloader.DownloadFileExceptionCb)(e =>
{
downloaded = false;
capturedException = e;
Logger.Error(e.ToString());
}));
if (!downloaded)
{
throw capturedException;
}
Manifest manifest = new Manifest(manifestFilePath);
manifest.Build();
return(manifest);
}
/// <summary>
/// Processes a single manifest file and all of the report keys it contains
/// </summary>
/// <param name="item"></param>
/// <param name="context"></param>
/// <returns></returns>
private static async Task <Manifest> ProcessItemAsync(S3EventNotificationRecord item, string destinationBucket, ILambdaContext context)
{
context.LogInfo(JsonConvert.SerializeObject(item));
// Make sure the event was when a new object was created
if (item.EventName != EventType.ObjectCreatedPut && item.EventName != EventType.ObjectCreatedPost)
{
string Message = $"This Lambda function was triggered by a non ObjectCreated Put or Post event, {item.EventName}, for object {item.S3.Object.Key}; check the CloudFormation template configuration and S3 Event setup.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(null);
}
// Get the manifest file contents
GetObjectRequest Request = new GetObjectRequest()
{
BucketName = item.S3.Bucket.Name,
Key = item.S3.Object.Key
};
string Body = "";
using (GetObjectResponse Response = await _S3Client.GetObjectAsync(Request))
{
using (Stream ResponseStream = Response.ResponseStream)
{
using (StreamReader Reader = new StreamReader(ResponseStream))
{
Body = await Reader.ReadToEndAsync();
}
}
}
Manifest ManifestFile = Manifest.Build(Body);
string Prefix = GetDestinationPrefix(ManifestFile);
// Build the destination key map to link source key to destination key
Dictionary <string, string> DestinationKeyMap = ManifestFile.ReportKeys.ToDictionary(x => x, x => $"{Prefix}/{Path.GetFileName(x)}");
// If there are no destination keys
// then there is nothing to do, return
if (!DestinationKeyMap.Any())
{
string Message = $"No destination keys produced for s3://{Request.BucketName}/{Request.Key}";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(null);
}
// Copy all of the files over first to replace existing files, this way there
// is no period of time where a file may not exist and break an active query
List <Task <CopyResponse> > CopyTasks = new List <Task <CopyResponse> >();
// Initiate a copy object task for each key
foreach (KeyValuePair <string, string> KeySet in DestinationKeyMap)
{
try
{
context.LogInfo($"Copying CUR from s3://{item.S3.Bucket.Name}/{KeySet.Key} to s3://{_DestinationBucket}/{KeySet.Value}");
CopyTasks.Add(CopyObjectAsync(KeySet.Key, KeySet.Value, item.S3.Bucket.Name, _DestinationBucket));
}
catch (Exception e)
{
string Message = $"Failed to add a copy object task to the queue for s3://{item.S3.Bucket.Name}/{KeySet.Key} to s3://{_DestinationBucket}/{KeySet.Value}.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
}
// Process the copy object results
foreach (Task <CopyResponse> Response in CopyTasks.Interleaved())
{
try
{
CopyResponse Result = await Response;
if (Result.IsError)
{
string Message = $"Failed to copy s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey}.";
context.LogError(Message, Result.Exception);
await SNSNotify(Message, context);
return(null);
}
else
{
if (Result.Response.HttpStatusCode != HttpStatusCode.OK)
{
string Message = $"Failed to copy s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey} with http code {(int)Result.Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(null);
}
else
{
context.LogInfo($"Successfully copied CUR from s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey}.");
}
}
}
catch (Exception e)
{
string Message = $"Internal error processing the copy async task.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
}
// Delete all of the keys in the that are not the files we just copied over
List <KeyVersion> KeysToDelete;
try
{
// Find all keys under the same prefix, and that aren't one of the keys of the files that have been copied
KeysToDelete = await ListAllObjectsAsync(destinationBucket, Prefix, x => x.Where(y => !DestinationKeyMap.Values.Contains(y.Key)));
}
catch (Exception e)
{
context.LogError(e);
await SNSNotify($"{e.Message}\n{e.StackTrace}", context);
return(null);
}
// Delete the old CUR files in the destination bucket
try
{
if (KeysToDelete != null && KeysToDelete.Any())
{
int DeletedCount = await DeleteObjectsAsync(KeysToDelete, destinationBucket);
if (DeletedCount != KeysToDelete.Count)
{
string Message = $"Unable to delete all objects, expected to delete {KeysToDelete.Count} but only deleted {DeletedCount}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(null);
}
else
{
context.LogInfo($"Successfully deleted {DeletedCount} objects.");
}
}
}
catch (Exception e)
{
string Message = "Unable to delete all old CUR files.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
return(ManifestFile);
}