public override void Execute() { ValidateRequest(); EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory)); ListObjectsRequest listRequest = ConstructListObjectRequest(); List<S3Object> objs = new List<S3Object>(); do { ListObjectsResponse listResponse = this._s3Client.ListObjects(listRequest); foreach (S3Object s3o in listResponse.S3Objects) { if (this._request.IsSetModifiedSinceDate() && s3o.LastModified <= this._request.ModifiedSinceDate) continue; if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified > this._request.UnmodifiedSinceDate) continue; this._totalBytes += s3o.Size; objs.Add(s3o); } listRequest.Marker = listResponse.NextMarker; } while (!string.IsNullOrEmpty(listRequest.Marker)); this._totalNumberOfFilesToDownload = objs.Count; foreach (S3Object s3o in objs) { if (s3o.Key.EndsWith("/", StringComparison.Ordinal)) continue; this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length); var downloadRequest = ConstructTransferUtilityDownloadRequest(s3o, listRequest.Prefix.Length); DownloadCommand command = new DownloadCommand(this._s3Client, downloadRequest); command.Execute(); } }
public override async Task ExecuteAsync(CancellationToken cancellationToken) { ValidateRequest(); EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory)); ListObjectsRequest listRequest = ConstructListObjectRequest(); List<S3Object> objs = new List<S3Object>(); do { ListObjectsResponse listResponse = await this._s3Client.ListObjectsAsync(listRequest, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); foreach (S3Object s3o in listResponse.S3Objects) { if (this._request.IsSetModifiedSinceDate() && s3o.LastModified <= this._request.ModifiedSinceDate) continue; if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified > this._request.UnmodifiedSinceDate) continue; this._totalBytes += s3o.Size; objs.Add(s3o); } listRequest.Marker = listResponse.NextMarker; } while (!string.IsNullOrEmpty(listRequest.Marker)); this._totalNumberOfFilesToDownload = objs.Count; SemaphoreSlim asyncThrottler = null; CancellationTokenSource internalCts = null; try { asyncThrottler = DownloadFilesConcurrently ? new SemaphoreSlim(this._config.ConcurrentServiceRequests) : new SemaphoreSlim(1); internalCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var pendingTasks = new List<Task>(); foreach (S3Object s3o in objs) { if (s3o.Key.EndsWith("/", StringComparison.Ordinal)) continue; await asyncThrottler.WaitAsync(cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); cancellationToken.ThrowIfCancellationRequested(); if (internalCts.IsCancellationRequested) { // Operation cancelled as one of the download requests failed with an exception, // don't schedule any more download tasks. // Don't throw an OperationCanceledException here as we want to process the // responses and throw the original exception. break; } // Valid for serial uploads when // TransferUtilityDownloadDirectoryRequest.DownloadFilesConcurrently is set to false. this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length); var downloadRequest = ConstructTransferUtilityDownloadRequest(s3o, listRequest.Prefix.Length); var command = new DownloadCommand(this._s3Client, downloadRequest); var task = ExecuteCommandAsync(command, internalCts, asyncThrottler); pendingTasks.Add(task); } await WhenAllOrFirstExceptionAsync(pendingTasks, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); } finally { internalCts.Dispose(); asyncThrottler.Dispose(); } }
/// <summary> /// Downloads the content from Amazon S3 and writes it to the specified file. /// If the key is not specified in the request parameter, /// the file name will be assumed. /// </summary> /// <param name="request"> /// Contains all the parameters used to download an Amazon S3 object. /// </param> public void Download(TransferUtilityDownloadRequest request) { BaseCommand command = new DownloadCommand(this._s3Client, request); command.Execute(); }
/// <summary> /// Initiates the asynchronous execution of the Download operation. /// <seealso cref="M:Amazon.S3.Transfer.TransferUtility.Download"/> /// </summary> /// <param name="request"> /// Contains all the parameters used to download an Amazon S3 object. /// </param> /// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param> /// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback procedure using the AsyncState property.</param> /// <exception cref="T:System.ArgumentNullException"></exception> /// <exception cref="T:System.Net.WebException"></exception> /// <exception cref="T:Amazon.S3.AmazonS3Exception"></exception> /// <returns>An IAsyncResult that can be used to poll or wait for results, or both; /// this value is also needed when invoking EndDownload.</returns> public IAsyncResult BeginDownload(TransferUtilityDownloadRequest request, AsyncCallback callback, object state) { BaseCommand command = new DownloadCommand(this._s3Client, request); return beginOperation(command, callback, state); }
public override void Execute() { if (!this._request.IsSetBucketName()) { throw new ArgumentNullException("BucketName", "The bucketName Specified is null or empty!"); } if (!this._request.IsSetS3Directory()) { throw new ArgumentNullException("S3Directory", "The S3Directory Specified is null or empty!"); } if (!this._request.IsSetLocalDirectory()) { throw new ArgumentNullException("LocalDirectory", "The LocalDirectory Specified is null or empty!"); } if (File.Exists(this._request.S3Directory)) { throw new ArgumentNullException("LocalDirectory", "A file already exists with the same name indicated by LocalDirectory!"); } EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory)); ListObjectsRequest listRequest = new ListObjectsRequest(); listRequest.BucketName = this._request.BucketName; listRequest.Prefix = this._request.S3Directory; listRequest.Prefix = listRequest.Prefix.Replace('\\', '/'); if (!listRequest.Prefix.EndsWith("/")) listRequest.Prefix += "/"; if (listRequest.Prefix.StartsWith("/")) { if (listRequest.Prefix.Length == 1) listRequest.Prefix = ""; else listRequest.Prefix = listRequest.Prefix.Substring(1); } ListObjectsResponse listResponse = this._s3Client.ListObjects(listRequest); List<S3Object> objs = new List<S3Object>(); foreach (S3Object s3o in listResponse.S3Objects) { DateTime lastModified = DateTime.Parse(s3o.LastModified); if (this._request.IsSetModifiedSinceDate() && this._request.ModifiedSinceDate < lastModified) continue; if (this._request.IsSetUnmodifiedSinceDate() && lastModified < this._request.UnmodifiedSinceDate) continue; objs.Add(s3o); } this._totalNumberOfFilesToDownload = objs.Count; foreach (S3Object s3o in objs) { this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length); TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest(); downloadRequest.BucketName = this._request.BucketName; downloadRequest.Key = s3o.Key; downloadRequest.FilePath = Path.Combine(this._request.LocalDirectory, this._currentFile); downloadRequest.Timeout = this._request.Timeout; downloadRequest.WithSubscriber(new EventHandler<WriteObjectProgressArgs>(downloadedProgressEventCallback)); DownloadCommand command = new DownloadCommand(this._s3Client, downloadRequest); command.Execute(); this._numberOfFilesDownloaded++; } }
public override async Task ExecuteAsync(CancellationToken cancellationToken) { ValidateRequest(); EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory)); ListObjectsRequest listRequest = ConstructListObjectRequest(); List <S3Object> objs = new List <S3Object>(); do { ListObjectsResponse listResponse = await this._s3Client.ListObjectsAsync(listRequest, cancellationToken). ConfigureAwait(continueOnCapturedContext: false); foreach (S3Object s3o in listResponse.S3Objects) { if (this._request.IsSetModifiedSinceDate() && s3o.LastModified <= this._request.ModifiedSinceDate) { continue; } if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified > this._request.UnmodifiedSinceDate) { continue; } this._totalBytes += s3o.Size; objs.Add(s3o); } listRequest.Marker = listResponse.NextMarker; } while (!string.IsNullOrEmpty(listRequest.Marker)); this._totalNumberOfFilesToDownload = objs.Count; SemaphoreSlim asyncThrottler = null; CancellationTokenSource internalCts = null; try { asyncThrottler = DownloadFilesConcurrently ? new SemaphoreSlim(this._config.ConcurrentServiceRequests) : new SemaphoreSlim(1); internalCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var pendingTasks = new List <Task>(); foreach (S3Object s3o in objs) { if (s3o.Key.EndsWith("/", StringComparison.Ordinal)) { continue; } await asyncThrottler.WaitAsync(cancellationToken). ConfigureAwait(continueOnCapturedContext: false); cancellationToken.ThrowIfCancellationRequested(); if (internalCts.IsCancellationRequested) { // Operation cancelled as one of the download requests failed with an exception, // don't schedule any more download tasks. // Don't throw an OperationCanceledException here as we want to process the // responses and throw the original exception. break; } // Valid for serial uploads when // TransferUtilityDownloadDirectoryRequest.DownloadFilesConcurrently is set to false. this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length); var downloadRequest = ConstructTransferUtilityDownloadRequest(s3o, listRequest.Prefix.Length); var command = new DownloadCommand(this._s3Client, downloadRequest); var task = ExecuteCommandAsync(command, internalCts, asyncThrottler); pendingTasks.Add(task); } await WhenAllOrFirstExceptionAsync(pendingTasks, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); } finally { internalCts.Dispose(); asyncThrottler.Dispose(); } }
/// <summary> /// Downloads the content from Amazon S3 and writes it to the specified file. /// If the key is not specified in the request parameter, /// the file name will used as the key name. /// </summary> /// <param name="request"> /// Contains all the parameters required to download an Amazon S3 object. /// </param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// <returns>The task object representing the asynchronous operation.</returns> public Task DownloadAsync(TransferUtilityDownloadRequest request, CancellationToken cancellationToken = default(CancellationToken)) { var command = new DownloadCommand(this._s3Client, request); return command.ExecuteAsync(cancellationToken); }
public override void Execute() { if (!this._request.IsSetBucketName()) { throw new InvalidOperationException("The bucketName Specified is null or empty!"); } if (!this._request.IsSetS3Directory()) { throw new InvalidOperationException("The S3Directory Specified is null or empty!"); } if (!this._request.IsSetLocalDirectory()) { throw new InvalidOperationException("The LocalDirectory Specified is null or empty!"); } if (File.Exists(this._request.S3Directory)) { throw new InvalidOperationException("A file already exists with the same name indicated by LocalDirectory!"); } EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory)); ListObjectsRequest listRequest = new ListObjectsRequest(); listRequest.BucketName = this._request.BucketName; listRequest.Prefix = this._request.S3Directory; listRequest.Prefix = listRequest.Prefix.Replace('\\', '/'); if (!listRequest.Prefix.EndsWith("/", StringComparison.Ordinal)) listRequest.Prefix += "/"; if (listRequest.Prefix.StartsWith("/", StringComparison.Ordinal)) { if (listRequest.Prefix.Length == 1) listRequest.Prefix = ""; else listRequest.Prefix = listRequest.Prefix.Substring(1); } List<S3Object> objs = new List<S3Object>(); do { ListObjectsResponse listResponse = this._s3Client.ListObjects(listRequest); foreach (S3Object s3o in listResponse.S3Objects) { if (this._request.IsSetModifiedSinceDate() && this._request.ModifiedSinceDate < s3o.LastModified) continue; if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified < this._request.UnmodifiedSinceDate) continue; objs.Add(s3o); } listRequest.Marker = listResponse.NextMarker; } while (!string.IsNullOrEmpty(listRequest.Marker)); this._totalNumberOfFilesToDownload = objs.Count; foreach (S3Object s3o in objs) { if (s3o.Key.EndsWith("/", StringComparison.Ordinal)) continue; this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length); TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest(); downloadRequest.BucketName = this._request.BucketName; downloadRequest.Key = s3o.Key; downloadRequest.FilePath = Path.Combine(this._request.LocalDirectory, this._currentFile); downloadRequest.WriteObjectProgressEvent += downloadedProgressEventCallback; DownloadCommand command = new DownloadCommand(this._s3Client, downloadRequest); command.Execute(); this._numberOfFilesDownloaded++; } }