private void btnCopy_Click(object sender, EventArgs e)
{
this.btnCopy.Enabled = false;
string fileName = string.Empty;
CopyResponse response = new CopyResponse();
if (tbFilename.Text.Trim() != string.Empty)
{
fileName = tbFilename.Text;
}
List <string> destFolders = new List <string>();
TargetFiles target = new TargetFiles();
target.FileName = fileName;
target.Load();
destFolders = target.TargetList;
response = new EACopyFactory(destFolders).Start();
if (response.Success)
{
lblStatus.Text = "Done !";
btnLoad.Enabled = true;
btnCopy.Enabled = false;
this.tbFilename.Text = string.Empty;
}
else
{
lblStatus.Text = response.Message;
btnLoad.Enabled = true;
btnCopy.Enabled = false;
this.tbFilename.Text = string.Empty;
}
}
private CopyResponse copyFolder(string sourceFolder, string destFolder)
{
CopyResponse response = new CopyResponse();
response.Success = true;
string[] files;
if (destFolder[destFolder.Length - 1] != Path.DirectorySeparatorChar)
{
destFolder += Path.DirectorySeparatorChar;
}
if (Directory.Exists(destFolder))
{
DeleteDirectory(destFolder, true);
}
if (!Directory.Exists(destFolder))
{
Directory.CreateDirectory(destFolder);
}
files = Directory.GetFileSystemEntries(sourceFolder);
try
{
foreach (string file in files)
{
if (Directory.Exists(file))
{
copyFolder(file, Path.Combine(destFolder, Path.GetFileName(file)));
}
else
{
File.Copy(file, Path.Combine(destFolder, Path.GetFileName(file)), true);
}
}
}
catch (Exception ex)
{
response.Message = ex.ToString();
response.Success = false;
}
return(response);
}
public CopyResponse Start()
{
string dropboxPath = GetDropBoxPath();
string mql4Path = Constants.MQL4;
string sourceFolder = Path.Combine(dropboxPath, mql4Path);
CopyResponse response = new CopyResponse();
if (_destFolders != null && _destFolders.Count() > 0)
{
foreach (string destFolder in _destFolders)
{
var destFolderMql = Path.Combine(destFolder, Constants.MQL4);
response = copyFolder(sourceFolder, destFolderMql);
if (response.Success)
{
WriteLog(Path.Combine(destFolder, Constants.MQL4));
}
}
}
return(response);
}
/// <summary>
/// Processes a single manifest file and all of the report keys it contains
/// </summary>
/// <param name="item"></param>
/// <param name="context"></param>
/// <returns></returns>
private static async Task <Manifest> ProcessItemAsync(S3EventNotificationRecord item, string destinationBucket, ILambdaContext context)
{
context.LogInfo(JsonConvert.SerializeObject(item));
// Make sure the event was when a new object was created
if (item.EventName != EventType.ObjectCreatedPut && item.EventName != EventType.ObjectCreatedPost)
{
string Message = $"This Lambda function was triggered by a non ObjectCreated Put or Post event, {item.EventName}, for object {item.S3.Object.Key}; check the CloudFormation template configuration and S3 Event setup.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(null);
}
// Get the manifest file contents
GetObjectRequest Request = new GetObjectRequest()
{
BucketName = item.S3.Bucket.Name,
Key = item.S3.Object.Key
};
string Body = "";
using (GetObjectResponse Response = await _S3Client.GetObjectAsync(Request))
{
using (Stream ResponseStream = Response.ResponseStream)
{
using (StreamReader Reader = new StreamReader(ResponseStream))
{
Body = await Reader.ReadToEndAsync();
}
}
}
Manifest ManifestFile = Manifest.Build(Body);
string Prefix = GetDestinationPrefix(ManifestFile);
// Build the destination key map to link source key to destination key
Dictionary <string, string> DestinationKeyMap = ManifestFile.ReportKeys.ToDictionary(x => x, x => $"{Prefix}/{Path.GetFileName(x)}");
// If there are no destination keys
// then there is nothing to do, return
if (!DestinationKeyMap.Any())
{
string Message = $"No destination keys produced for s3://{Request.BucketName}/{Request.Key}";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(null);
}
// Copy all of the files over first to replace existing files, this way there
// is no period of time where a file may not exist and break an active query
List <Task <CopyResponse> > CopyTasks = new List <Task <CopyResponse> >();
// Initiate a copy object task for each key
foreach (KeyValuePair <string, string> KeySet in DestinationKeyMap)
{
try
{
context.LogInfo($"Copying CUR from s3://{item.S3.Bucket.Name}/{KeySet.Key} to s3://{_DestinationBucket}/{KeySet.Value}");
CopyTasks.Add(CopyObjectAsync(KeySet.Key, KeySet.Value, item.S3.Bucket.Name, _DestinationBucket));
}
catch (Exception e)
{
string Message = $"Failed to add a copy object task to the queue for s3://{item.S3.Bucket.Name}/{KeySet.Key} to s3://{_DestinationBucket}/{KeySet.Value}.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
}
// Process the copy object results
foreach (Task <CopyResponse> Response in CopyTasks.Interleaved())
{
try
{
CopyResponse Result = await Response;
if (Result.IsError)
{
string Message = $"Failed to copy s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey}.";
context.LogError(Message, Result.Exception);
await SNSNotify(Message, context);
return(null);
}
else
{
if (Result.Response.HttpStatusCode != HttpStatusCode.OK)
{
string Message = $"Failed to copy s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey} with http code {(int)Result.Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(null);
}
else
{
context.LogInfo($"Successfully copied CUR from s3://{Result.SourceBucket}/{Result.SourceKey} to s3://{Result.DestinationBucket}/{Result.DestinationKey}.");
}
}
}
catch (Exception e)
{
string Message = $"Internal error processing the copy async task.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
}
// Delete all of the keys in the that are not the files we just copied over
List <KeyVersion> KeysToDelete;
try
{
// Find all keys under the same prefix, and that aren't one of the keys of the files that have been copied
KeysToDelete = await ListAllObjectsAsync(destinationBucket, Prefix, x => x.Where(y => !DestinationKeyMap.Values.Contains(y.Key)));
}
catch (Exception e)
{
context.LogError(e);
await SNSNotify($"{e.Message}\n{e.StackTrace}", context);
return(null);
}
// Delete the old CUR files in the destination bucket
try
{
if (KeysToDelete != null && KeysToDelete.Any())
{
int DeletedCount = await DeleteObjectsAsync(KeysToDelete, destinationBucket);
if (DeletedCount != KeysToDelete.Count)
{
string Message = $"Unable to delete all objects, expected to delete {KeysToDelete.Count} but only deleted {DeletedCount}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(null);
}
else
{
context.LogInfo($"Successfully deleted {DeletedCount} objects.");
}
}
}
catch (Exception e)
{
string Message = "Unable to delete all old CUR files.";
context.LogError(Message, e);
await SNSNotify(Message, context);
return(null);
}
return(ManifestFile);
}
