public S3TransferAgent(TransferAgentOptions options, AWSCredentials awsCredentials, string awsBucketName, CancellationToken cancellationToken)
: base(options, new S3StorageBackend(options, awsCredentials, GetDefaultS3Config(), awsBucketName), cancellationToken)
{
// This is `true` because it should dispose the `StorageBackend` implementation passed to the base class.
_shouldDispose = true;
PathBuilder = new S3PathBuilder();
}
public override void Start()
{
Assert.IsFalse(IsRunning);
Assert.IsNotNull(Synchronization);
Assert.IsNotNull(Synchronization.StorageAccount);
Assert.AreEqual(Models.EStorageAccountType.AmazonS3, Synchronization.StorageAccountType);
AmazonS3AccountRepository dao = new AmazonS3AccountRepository();
Models.AmazonS3Account s3account = dao.GetForReadOnly(Synchronization.StorageAccount.Id);
//
// Dispose and recycle previous objects, if needed.
//
if (TransferAgent != null)
{
TransferAgent.Dispose();
}
if (TransferListControl != null)
{
TransferListControl.ClearTransfers();
}
if (SyncAgent != null)
{
SyncAgent.Dispose();
}
//
// Setup agents.
//
AWSCredentials awsCredentials = new BasicAWSCredentials(s3account.AccessKey, s3account.SecretKey);
TransferAgentOptions options = new TransferAgentOptions
{
UploadChunkSizeInBytes = Teltec.Everest.Settings.Properties.Current.UploadChunkSize * 1024 * 1024,
};
TransferAgent = new S3TransferAgent(options, awsCredentials, s3account.BucketName, CancellationTokenSource.Token);
TransferAgent.RemoteRootDir = TransferAgent.PathBuilder.CombineRemotePath("TELTEC_BKP", s3account.Hostname);
RemoteObjects = new List <ListingObject>(4096); // Avoid small resizes without compromising memory.
SyncAgent = new CustomSynchronizationAgent(TransferAgent);
RegisterEventHandlers(Synchronization);
Report.PlanType = "synchronization";
Report.PlanName = "No plan";
Report.BucketName = s3account.BucketName;
Report.HostName = Synchronization.StorageAccount.Hostname;
Report.SyncResults = SyncAgent.Results;
//
// Start the sync.
//
DoSynchronization(SyncAgent, Synchronization, Options);
}
public S3StorageBackend(TransferAgentOptions options, AWSCredentials awsCredentials, AmazonS3Config s3Config, string awsBucketName)
{
this._shouldDispose = true;
Options = options;
SanitizeOptions();
this._s3Config = s3Config;
this._s3Config.RegionEndpoint = GetBucketEndpoint(awsCredentials, awsBucketName);
this._s3Client = new AmazonS3Client(awsCredentials, s3Config);
this._awsBuckeName = awsBucketName;
}
public override void Start()
{
Assert.IsFalse(IsRunning);
Assert.IsNotNull(Restore);
Assert.IsNotNull(Restore.RestorePlan.StorageAccount);
Assert.AreEqual(Models.EStorageAccountType.AmazonS3, Restore.RestorePlan.StorageAccountType);
AmazonS3AccountRepository dao = new AmazonS3AccountRepository();
Models.AmazonS3Account s3account = dao.GetForReadOnly(Restore.RestorePlan.StorageAccount.Id);
//
// Dispose and recycle previous objects, if needed.
//
if (TransferAgent != null)
{
TransferAgent.Dispose();
}
if (TransferListControl != null)
{
TransferListControl.ClearTransfers();
}
if (RestoreAgent != null)
{
RestoreAgent.Dispose();
}
if (Versioner != null)
{
Versioner.Dispose();
}
//
// Setup agents.
//
AWSCredentials awsCredentials = new BasicAWSCredentials(s3account.AccessKey, s3account.SecretKey);
TransferAgentOptions options = new TransferAgentOptions
{
UploadChunkSizeInBytes = Teltec.Everest.Settings.Properties.Current.UploadChunkSize * 1024 * 1024,
};
TransferAgent = new S3TransferAgent(options, awsCredentials, s3account.BucketName, CancellationTokenSource.Token);
TransferAgent.RemoteRootDir = TransferAgent.PathBuilder.CombineRemotePath("TELTEC_BKP",
Restore.RestorePlan.StorageAccount.Hostname);
RestoreAgent = new CustomRestoreAgent(TransferAgent);
RestoreAgent.Results.Monitor = TransferListControl;
Versioner = new RestoreFileVersioner(CancellationTokenSource.Token);
RegisterResultsEventHandlers(Restore, RestoreAgent.Results);
Report.PlanType = "restore";
Report.PlanName = Restore.RestorePlan.Name;
Report.BucketName = s3account.BucketName;
Report.HostName = Restore.RestorePlan.StorageAccount.Hostname;
Report.TransferResults = RestoreAgent.Results;
Helper = new BaseOperationHelper(Restore.RestorePlan);
//
// Start the restore.
//
DoRestore(RestoreAgent, Restore, Options);
}
private async void DoRun()
{
_IsRunning = true;
SaveSettings();
AsyncHelper.SettingsMaxThreadCount = Decimal.ToInt32(nudParallelism.Value);
if (_TransferAgent != null)
{
_TransferAgent.Dispose();
}
transferListControl1.ClearTransfers();
AWSCredentials awsCredentials = new BasicAWSCredentials(tbAccessKey.Text, tbSecretKey.Text);
TransferAgentOptions options = new TransferAgentOptions();
if (CancelTokenSource != null)
{
CancelTokenSource.Dispose();
}
CancelTokenSource = new CancellationTokenSource();
_TransferAgent = new S3TransferAgent(options, awsCredentials, tbBucketName.Text, CancelTokenSource.Token);
_TransferAgent.RemoteRootDir = "backup-99";
switch (_Operation)
{
case OperationType.BACKUP:
{
_BackupAgent = new CustomBackupAgent(_TransferAgent);
_BackupAgent.Results.Monitor = transferListControl1;
/*
* _BackupAgent.Results.Failed += (object sender, TransferFileProgressArgs args, Exception e) =>
* {
* Warn("Failed {0}", args.FilePath);
* };
* _BackupAgent.Results.Canceled += (object sender, TransferFileProgressArgs args, Exception e) =>
* {
* Warn("Canceled {0}", args.FilePath);
* };
* _BackupAgent.Results.Completed += (object sender, TransferFileProgressArgs args) =>
* {
* Info("Completed {0}", args.FilePath);
* };
* _BackupAgent.Results.Started += (object sender, TransferFileProgressArgs args) =>
* {
* Info("Started {0}", args.FilePath);
* };
* _BackupAgent.Results.Progress += (object sender, TransferFileProgressArgs args) =>
* {
* Info("Progress {0}% {1} ({2}/{3} bytes)",
* args.PercentDone, args.FilePath, args.TransferredBytes, args.TotalBytes);
* };
*/
LinkedList <CustomVersionedFile> sources = new LinkedList <CustomVersionedFile>();
if (cbSimulateFailure.Checked)
{
sources.AddLast(new CustomVersionedFile(@"C:\pagefile.sys"));
}
DirectoryInfo dir = new DirectoryInfo(txtSourceDirectory.Text);
if (dir != null)
{
foreach (FileInfo file in dir.GetFiles())
{
sources.AddLast(new CustomVersionedFile(file.FullName));
}
}
_BackupAgent.Files = sources;
Info("Estimate backup size: {0} files, {1} bytes",
_BackupAgent.Results.Stats.Total,
FileSizeUtils.FileSizeToString(_BackupAgent.EstimatedTransferSize));
Task task = _BackupAgent.Start();
try
{
await task;
}
catch (Exception ex)
{
if (ex.IsCancellation())
{
Info(ex.Message);
}
else
{
Error(ex.Message);
}
}
break;
}
case OperationType.RESTORE:
{
_RestoreAgent = new CustomRestoreAgent(_TransferAgent);
_RestoreAgent.Results.Monitor = transferListControl1;
// TODO(jweyrich): These are statically hardcoded for now, but they should be dynamic.
// To make them dynamic we need to execute a Sync operation to discover them first.
LinkedList <CustomVersionedFile> sources = new LinkedList <CustomVersionedFile>();
sources.AddLast(new CustomVersionedFile(@"C:\pagefile.sys"));
sources.AddLast(new CustomVersionedFile(@"C:\Teste\a.txt"));
sources.AddLast(new CustomVersionedFile(@"C:\Teste\b.txt"));
sources.AddLast(new CustomVersionedFile(@"C:\Teste\bash.exe"));
sources.AddLast(new CustomVersionedFile(@"C:\Teste\c.txt"));
sources.AddLast(new CustomVersionedFile(@"C:\Teste\e.txt"));
_RestoreAgent.Files = sources;
Info("Estimate backup size: {0} files, {1} bytes",
_RestoreAgent.Results.Stats.Total,
FileSizeUtils.FileSizeToString(_RestoreAgent.EstimatedTransferSize));
Task task = _RestoreAgent.Start();
try
{
await task;
}
catch (Exception ex)
{
if (ex.IsCancellation())
{
Info(ex.Message);
}
else
{
Error(ex.Message);
}
}
break;
}
}
OnFinish();
}
private void btnStart_Click(object sender, EventArgs e)
{
if (IsRunning)
{
return;
}
IsRunning = true;
CancellationTokenSource = new CancellationTokenSource();
var options = new TransferAgentOptions
{
UploadChunkSizeInBytes = 1 * 1024 * 1024,
};
string accessKey = txtAccessKey.Text.Trim();
string secretKey = txtSecretKey.Text.Trim();
string bucketName = txtBucketName.Text.Trim();
BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
string localFilePath = txtFilePath.Text;
bool fileInformed = !string.IsNullOrEmpty(localFilePath);
bool fileExists = fileInformed && FileManager.FileExists(localFilePath);
if (!fileInformed || !fileExists)
{
string message = "";
if (!fileInformed)
{
message = "You have to inform a file for upload";
}
else if (!fileExists)
{
message = string.Format("The informed file does not exist: {0}", localFilePath);
}
MessageBox.Show(message, "Oops!", MessageBoxButtons.OK, MessageBoxIcon.Error);
IsRunning = false;
return;
}
#if true
string remoteFilePath = typeof(UploadPerfTestControl).Name + ".DELETE_ME";
#else
S3PathBuilder pathBuilder = new S3PathBuilder();
string remoteFilePath = pathBuilder.BuildRemotePath(localFilePath);
#endif
long fileSize = FileManager.UnsafeGetFileSize(localFilePath);
BlockPerfStats stats = new BlockPerfStats();
S3TransferAgent xferAgent = new S3TransferAgent(options, credentials, bucketName, CancellationTokenSource.Token);
xferAgent.UploadFileStarted += (object sender1, TransferFileProgressArgs e1) =>
{
stats.Begin();
};
xferAgent.UploadFileCanceled += (object sender1, TransferFileProgressArgs e1) =>
{
stats.End();
string message = "Canceled file upload";
MessageBox.Show(message, "Transfer canceled", MessageBoxButtons.OK, MessageBoxIcon.Information);
};
xferAgent.UploadFileFailed += (object sender1, TransferFileProgressArgs e1) =>
{
stats.End();
string message = string.Format("Failed to upload file: {0}\n{1}", e1.Exception.GetType().Name, e1.Exception.Message);
MessageBox.Show(message, "Transfer failed", MessageBoxButtons.OK, MessageBoxIcon.Error);
};
xferAgent.UploadFileCompleted += (object sender1, TransferFileProgressArgs e1) =>
{
stats.End();
string message = string.Format(
"Took {0} to upload {1}",
TimeSpanUtils.GetReadableTimespan(stats.Duration),
FileSizeUtils.FileSizeToString(fileSize)
);
MessageBox.Show(message, "Transfer completed", MessageBoxButtons.OK, MessageBoxIcon.Information);
};
//xferAgent.UploadFileProgress += (object sender1, TransferFileProgressArgs e1) =>
//{
// // ...
//};
xferAgent.UploadFile(localFilePath, remoteFilePath, null);
IsRunning = false;
}
private async Task <List <string> > RetrieveAccountConfigurations()
{
// Renew cancellation token if was already used.
if (CancellationTokenSource.IsCancellationRequested)
{
CancellationTokenSource.Dispose();
CancellationTokenSource = new CancellationTokenSource();
}
//
// Setup agents.
//
AWSCredentials awsCredentials = new BasicAWSCredentials(_account.AccessKey, _account.SecretKey);
TransferAgentOptions options = new TransferAgentOptions();
ITransferAgent transferAgent = new S3TransferAgent(options, awsCredentials, _account.BucketName, CancellationTokenSource.Token);
transferAgent.RemoteRootDir = transferAgent.PathBuilder.CombineRemotePath("TELTEC_BKP");
List <string> remoteObjects = new List <string>(16); // Avoid small resizes without compromising memory.
// Register event handlers.
transferAgent.ListingProgress += (object sender2, ListingProgressArgs e2) =>
{
foreach (var obj in e2.Objects)
{
string[] parts = obj.Key.Split(S3PathBuilder.RemoteDirectorySeparatorChar);
int count = parts.Length;
if (count > 2)
{
remoteObjects.Add(parts[count - 2]);
}
}
};
transferAgent.ListingFailed += (object sender2, ListingProgressArgs e2) =>
{
//var message = string.Format("Failed: {0}", e2.Exception != null ? e2.Exception.Message : "Unknown reason");
//logger.Warn(message);
};
try
{
Task task = Task.Run(() =>
{
transferAgent.List(transferAgent.RemoteRootDir, false, null);
});
await task;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception)
{
throw;
}
finally
{
}
return(remoteObjects);
}
