public S3TransferAgent(TransferAgentOptions options, AWSCredentials awsCredentials, string awsBucketName, CancellationToken cancellationToken) : base(options, new S3StorageBackend(options, awsCredentials, GetDefaultS3Config(), awsBucketName), cancellationToken) { // This is `true` because it should dispose the `StorageBackend` implementation passed to the base class. _shouldDispose = true; PathBuilder = new S3PathBuilder(); }
public override void Start() { Assert.IsFalse(IsRunning); Assert.IsNotNull(Synchronization); Assert.IsNotNull(Synchronization.StorageAccount); Assert.AreEqual(Models.EStorageAccountType.AmazonS3, Synchronization.StorageAccountType); AmazonS3AccountRepository dao = new AmazonS3AccountRepository(); Models.AmazonS3Account s3account = dao.GetForReadOnly(Synchronization.StorageAccount.Id); // // Dispose and recycle previous objects, if needed. // if (TransferAgent != null) { TransferAgent.Dispose(); } if (TransferListControl != null) { TransferListControl.ClearTransfers(); } if (SyncAgent != null) { SyncAgent.Dispose(); } // // Setup agents. // AWSCredentials awsCredentials = new BasicAWSCredentials(s3account.AccessKey, s3account.SecretKey); TransferAgentOptions options = new TransferAgentOptions { UploadChunkSizeInBytes = Teltec.Everest.Settings.Properties.Current.UploadChunkSize * 1024 * 1024, }; TransferAgent = new S3TransferAgent(options, awsCredentials, s3account.BucketName, CancellationTokenSource.Token); TransferAgent.RemoteRootDir = TransferAgent.PathBuilder.CombineRemotePath("TELTEC_BKP", s3account.Hostname); RemoteObjects = new List <ListingObject>(4096); // Avoid small resizes without compromising memory. SyncAgent = new CustomSynchronizationAgent(TransferAgent); RegisterEventHandlers(Synchronization); Report.PlanType = "synchronization"; Report.PlanName = "No plan"; Report.BucketName = s3account.BucketName; Report.HostName = Synchronization.StorageAccount.Hostname; Report.SyncResults = SyncAgent.Results; // // Start the sync. // DoSynchronization(SyncAgent, Synchronization, Options); }
public S3StorageBackend(TransferAgentOptions options, AWSCredentials awsCredentials, AmazonS3Config s3Config, string awsBucketName) { this._shouldDispose = true; Options = options; SanitizeOptions(); this._s3Config = s3Config; this._s3Config.RegionEndpoint = GetBucketEndpoint(awsCredentials, awsBucketName); this._s3Client = new AmazonS3Client(awsCredentials, s3Config); this._awsBuckeName = awsBucketName; }
public override void Start() { Assert.IsFalse(IsRunning); Assert.IsNotNull(Restore); Assert.IsNotNull(Restore.RestorePlan.StorageAccount); Assert.AreEqual(Models.EStorageAccountType.AmazonS3, Restore.RestorePlan.StorageAccountType); AmazonS3AccountRepository dao = new AmazonS3AccountRepository(); Models.AmazonS3Account s3account = dao.GetForReadOnly(Restore.RestorePlan.StorageAccount.Id); // // Dispose and recycle previous objects, if needed. // if (TransferAgent != null) { TransferAgent.Dispose(); } if (TransferListControl != null) { TransferListControl.ClearTransfers(); } if (RestoreAgent != null) { RestoreAgent.Dispose(); } if (Versioner != null) { Versioner.Dispose(); } // // Setup agents. // AWSCredentials awsCredentials = new BasicAWSCredentials(s3account.AccessKey, s3account.SecretKey); TransferAgentOptions options = new TransferAgentOptions { UploadChunkSizeInBytes = Teltec.Everest.Settings.Properties.Current.UploadChunkSize * 1024 * 1024, }; TransferAgent = new S3TransferAgent(options, awsCredentials, s3account.BucketName, CancellationTokenSource.Token); TransferAgent.RemoteRootDir = TransferAgent.PathBuilder.CombineRemotePath("TELTEC_BKP", Restore.RestorePlan.StorageAccount.Hostname); RestoreAgent = new CustomRestoreAgent(TransferAgent); RestoreAgent.Results.Monitor = TransferListControl; Versioner = new RestoreFileVersioner(CancellationTokenSource.Token); RegisterResultsEventHandlers(Restore, RestoreAgent.Results); Report.PlanType = "restore"; Report.PlanName = Restore.RestorePlan.Name; Report.BucketName = s3account.BucketName; Report.HostName = Restore.RestorePlan.StorageAccount.Hostname; Report.TransferResults = RestoreAgent.Results; Helper = new BaseOperationHelper(Restore.RestorePlan); // // Start the restore. // DoRestore(RestoreAgent, Restore, Options); }
private async void DoRun() { _IsRunning = true; SaveSettings(); AsyncHelper.SettingsMaxThreadCount = Decimal.ToInt32(nudParallelism.Value); if (_TransferAgent != null) { _TransferAgent.Dispose(); } transferListControl1.ClearTransfers(); AWSCredentials awsCredentials = new BasicAWSCredentials(tbAccessKey.Text, tbSecretKey.Text); TransferAgentOptions options = new TransferAgentOptions(); if (CancelTokenSource != null) { CancelTokenSource.Dispose(); } CancelTokenSource = new CancellationTokenSource(); _TransferAgent = new S3TransferAgent(options, awsCredentials, tbBucketName.Text, CancelTokenSource.Token); _TransferAgent.RemoteRootDir = "backup-99"; switch (_Operation) { case OperationType.BACKUP: { _BackupAgent = new CustomBackupAgent(_TransferAgent); _BackupAgent.Results.Monitor = transferListControl1; /* * _BackupAgent.Results.Failed += (object sender, TransferFileProgressArgs args, Exception e) => * { * Warn("Failed {0}", args.FilePath); * }; * _BackupAgent.Results.Canceled += (object sender, TransferFileProgressArgs args, Exception e) => * { * Warn("Canceled {0}", args.FilePath); * }; * _BackupAgent.Results.Completed += (object sender, TransferFileProgressArgs args) => * { * Info("Completed {0}", args.FilePath); * }; * _BackupAgent.Results.Started += (object sender, TransferFileProgressArgs args) => * { * Info("Started {0}", args.FilePath); * }; * _BackupAgent.Results.Progress += (object sender, TransferFileProgressArgs args) => * { * Info("Progress {0}% {1} ({2}/{3} bytes)", * args.PercentDone, args.FilePath, args.TransferredBytes, args.TotalBytes); * }; */ LinkedList <CustomVersionedFile> sources = new LinkedList <CustomVersionedFile>(); if (cbSimulateFailure.Checked) { sources.AddLast(new CustomVersionedFile(@"C:\pagefile.sys")); } DirectoryInfo dir = new DirectoryInfo(txtSourceDirectory.Text); if (dir != null) { foreach (FileInfo file in dir.GetFiles()) { sources.AddLast(new CustomVersionedFile(file.FullName)); } } _BackupAgent.Files = sources; Info("Estimate backup size: {0} files, {1} bytes", _BackupAgent.Results.Stats.Total, FileSizeUtils.FileSizeToString(_BackupAgent.EstimatedTransferSize)); Task task = _BackupAgent.Start(); try { await task; } catch (Exception ex) { if (ex.IsCancellation()) { Info(ex.Message); } else { Error(ex.Message); } } break; } case OperationType.RESTORE: { _RestoreAgent = new CustomRestoreAgent(_TransferAgent); _RestoreAgent.Results.Monitor = transferListControl1; // TODO(jweyrich): These are statically hardcoded for now, but they should be dynamic. // To make them dynamic we need to execute a Sync operation to discover them first. LinkedList <CustomVersionedFile> sources = new LinkedList <CustomVersionedFile>(); sources.AddLast(new CustomVersionedFile(@"C:\pagefile.sys")); sources.AddLast(new CustomVersionedFile(@"C:\Teste\a.txt")); sources.AddLast(new CustomVersionedFile(@"C:\Teste\b.txt")); sources.AddLast(new CustomVersionedFile(@"C:\Teste\bash.exe")); sources.AddLast(new CustomVersionedFile(@"C:\Teste\c.txt")); sources.AddLast(new CustomVersionedFile(@"C:\Teste\e.txt")); _RestoreAgent.Files = sources; Info("Estimate backup size: {0} files, {1} bytes", _RestoreAgent.Results.Stats.Total, FileSizeUtils.FileSizeToString(_RestoreAgent.EstimatedTransferSize)); Task task = _RestoreAgent.Start(); try { await task; } catch (Exception ex) { if (ex.IsCancellation()) { Info(ex.Message); } else { Error(ex.Message); } } break; } } OnFinish(); }
private void btnStart_Click(object sender, EventArgs e) { if (IsRunning) { return; } IsRunning = true; CancellationTokenSource = new CancellationTokenSource(); var options = new TransferAgentOptions { UploadChunkSizeInBytes = 1 * 1024 * 1024, }; string accessKey = txtAccessKey.Text.Trim(); string secretKey = txtSecretKey.Text.Trim(); string bucketName = txtBucketName.Text.Trim(); BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); string localFilePath = txtFilePath.Text; bool fileInformed = !string.IsNullOrEmpty(localFilePath); bool fileExists = fileInformed && FileManager.FileExists(localFilePath); if (!fileInformed || !fileExists) { string message = ""; if (!fileInformed) { message = "You have to inform a file for upload"; } else if (!fileExists) { message = string.Format("The informed file does not exist: {0}", localFilePath); } MessageBox.Show(message, "Oops!", MessageBoxButtons.OK, MessageBoxIcon.Error); IsRunning = false; return; } #if true string remoteFilePath = typeof(UploadPerfTestControl).Name + ".DELETE_ME"; #else S3PathBuilder pathBuilder = new S3PathBuilder(); string remoteFilePath = pathBuilder.BuildRemotePath(localFilePath); #endif long fileSize = FileManager.UnsafeGetFileSize(localFilePath); BlockPerfStats stats = new BlockPerfStats(); S3TransferAgent xferAgent = new S3TransferAgent(options, credentials, bucketName, CancellationTokenSource.Token); xferAgent.UploadFileStarted += (object sender1, TransferFileProgressArgs e1) => { stats.Begin(); }; xferAgent.UploadFileCanceled += (object sender1, TransferFileProgressArgs e1) => { stats.End(); string message = "Canceled file upload"; MessageBox.Show(message, "Transfer canceled", MessageBoxButtons.OK, MessageBoxIcon.Information); }; xferAgent.UploadFileFailed += (object sender1, TransferFileProgressArgs e1) => { stats.End(); string message = string.Format("Failed to upload file: {0}\n{1}", e1.Exception.GetType().Name, e1.Exception.Message); MessageBox.Show(message, "Transfer failed", MessageBoxButtons.OK, MessageBoxIcon.Error); }; xferAgent.UploadFileCompleted += (object sender1, TransferFileProgressArgs e1) => { stats.End(); string message = string.Format( "Took {0} to upload {1}", TimeSpanUtils.GetReadableTimespan(stats.Duration), FileSizeUtils.FileSizeToString(fileSize) ); MessageBox.Show(message, "Transfer completed", MessageBoxButtons.OK, MessageBoxIcon.Information); }; //xferAgent.UploadFileProgress += (object sender1, TransferFileProgressArgs e1) => //{ // // ... //}; xferAgent.UploadFile(localFilePath, remoteFilePath, null); IsRunning = false; }
private async Task <List <string> > RetrieveAccountConfigurations() { // Renew cancellation token if was already used. if (CancellationTokenSource.IsCancellationRequested) { CancellationTokenSource.Dispose(); CancellationTokenSource = new CancellationTokenSource(); } // // Setup agents. // AWSCredentials awsCredentials = new BasicAWSCredentials(_account.AccessKey, _account.SecretKey); TransferAgentOptions options = new TransferAgentOptions(); ITransferAgent transferAgent = new S3TransferAgent(options, awsCredentials, _account.BucketName, CancellationTokenSource.Token); transferAgent.RemoteRootDir = transferAgent.PathBuilder.CombineRemotePath("TELTEC_BKP"); List <string> remoteObjects = new List <string>(16); // Avoid small resizes without compromising memory. // Register event handlers. transferAgent.ListingProgress += (object sender2, ListingProgressArgs e2) => { foreach (var obj in e2.Objects) { string[] parts = obj.Key.Split(S3PathBuilder.RemoteDirectorySeparatorChar); int count = parts.Length; if (count > 2) { remoteObjects.Add(parts[count - 2]); } } }; transferAgent.ListingFailed += (object sender2, ListingProgressArgs e2) => { //var message = string.Format("Failed: {0}", e2.Exception != null ? e2.Exception.Message : "Unknown reason"); //logger.Warn(message); }; try { Task task = Task.Run(() => { transferAgent.List(transferAgent.RemoteRootDir, false, null); }); await task; } catch (OperationCanceledException) { throw; } catch (Exception) { throw; } finally { } return(remoteObjects); }