public override void Execute()
{
ValidateRequest();
EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory));
ListObjectsRequest listRequest = ConstructListObjectRequest();
List<S3Object> objs = new List<S3Object>();
do
{
ListObjectsResponse listResponse = this._s3Client.ListObjects(listRequest);
foreach (S3Object s3o in listResponse.S3Objects)
{
if (this._request.IsSetModifiedSinceDate() && s3o.LastModified <= this._request.ModifiedSinceDate)
continue;
if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified > this._request.UnmodifiedSinceDate)
continue;
this._totalBytes += s3o.Size;
objs.Add(s3o);
}
listRequest.Marker = listResponse.NextMarker;
} while (!string.IsNullOrEmpty(listRequest.Marker));
this._totalNumberOfFilesToDownload = objs.Count;
foreach (S3Object s3o in objs)
{
if (s3o.Key.EndsWith("/", StringComparison.Ordinal))
continue;
this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length);
var downloadRequest = ConstructTransferUtilityDownloadRequest(s3o, listRequest.Prefix.Length);
DownloadCommand command = new DownloadCommand(this._s3Client, downloadRequest);
command.Execute();
}
}
public override async Task ExecuteAsync(CancellationToken cancellationToken)
{
ValidateRequest();
EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory));
ListObjectsRequest listRequest = ConstructListObjectRequest();
List<S3Object> objs = new List<S3Object>();
do
{
ListObjectsResponse listResponse = await this._s3Client.ListObjectsAsync(listRequest, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
foreach (S3Object s3o in listResponse.S3Objects)
{
if (this._request.IsSetModifiedSinceDate() && s3o.LastModified <= this._request.ModifiedSinceDate)
continue;
if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified > this._request.UnmodifiedSinceDate)
continue;
this._totalBytes += s3o.Size;
objs.Add(s3o);
}
listRequest.Marker = listResponse.NextMarker;
} while (!string.IsNullOrEmpty(listRequest.Marker));
this._totalNumberOfFilesToDownload = objs.Count;
SemaphoreSlim asyncThrottler = null;
CancellationTokenSource internalCts = null;
try
{
asyncThrottler = DownloadFilesConcurrently ?
new SemaphoreSlim(this._config.ConcurrentServiceRequests) :
new SemaphoreSlim(1);
internalCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var pendingTasks = new List<Task>();
foreach (S3Object s3o in objs)
{
if (s3o.Key.EndsWith("/", StringComparison.Ordinal))
continue;
await asyncThrottler.WaitAsync(cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
cancellationToken.ThrowIfCancellationRequested();
if (internalCts.IsCancellationRequested)
{
// Operation cancelled as one of the download requests failed with an exception,
// don't schedule any more download tasks.
// Don't throw an OperationCanceledException here as we want to process the
// responses and throw the original exception.
break;
}
// Valid for serial uploads when
// TransferUtilityDownloadDirectoryRequest.DownloadFilesConcurrently is set to false.
this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length);
var downloadRequest = ConstructTransferUtilityDownloadRequest(s3o, listRequest.Prefix.Length);
var command = new DownloadCommand(this._s3Client, downloadRequest);
var task = ExecuteCommandAsync(command, internalCts, asyncThrottler);
pendingTasks.Add(task);
}
await WhenAllOrFirstExceptionAsync(pendingTasks, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
}
finally
{
internalCts.Dispose();
asyncThrottler.Dispose();
}
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// If the key is not specified in the request parameter,
/// the file name will be assumed.
/// </summary>
/// <param name="request">
/// Contains all the parameters used to download an Amazon S3 object.
/// </param>
public void Download(TransferUtilityDownloadRequest request)
{
BaseCommand command = new DownloadCommand(this._s3Client, request);
command.Execute();
}
/// <summary>
/// Initiates the asynchronous execution of the Download operation.
/// <seealso cref="M:Amazon.S3.Transfer.TransferUtility.Download"/>
/// </summary>
/// <param name="request">
/// Contains all the parameters used to download an Amazon S3 object.
/// </param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback procedure using the AsyncState property.</param>
/// <exception cref="T:System.ArgumentNullException"></exception>
/// <exception cref="T:System.Net.WebException"></exception>
/// <exception cref="T:Amazon.S3.AmazonS3Exception"></exception>
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both;
/// this value is also needed when invoking EndDownload.</returns>
public IAsyncResult BeginDownload(TransferUtilityDownloadRequest request, AsyncCallback callback, object state)
{
BaseCommand command = new DownloadCommand(this._s3Client, request);
return beginOperation(command, callback, state);
}
public override void Execute()
{
if (!this._request.IsSetBucketName())
{
throw new ArgumentNullException("BucketName", "The bucketName Specified is null or empty!");
}
if (!this._request.IsSetS3Directory())
{
throw new ArgumentNullException("S3Directory", "The S3Directory Specified is null or empty!");
}
if (!this._request.IsSetLocalDirectory())
{
throw new ArgumentNullException("LocalDirectory", "The LocalDirectory Specified is null or empty!");
}
if (File.Exists(this._request.S3Directory))
{
throw new ArgumentNullException("LocalDirectory", "A file already exists with the same name indicated by LocalDirectory!");
}
EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory));
ListObjectsRequest listRequest = new ListObjectsRequest();
listRequest.BucketName = this._request.BucketName;
listRequest.Prefix = this._request.S3Directory;
listRequest.Prefix = listRequest.Prefix.Replace('\\', '/');
if (!listRequest.Prefix.EndsWith("/"))
listRequest.Prefix += "/";
if (listRequest.Prefix.StartsWith("/"))
{
if (listRequest.Prefix.Length == 1)
listRequest.Prefix = "";
else
listRequest.Prefix = listRequest.Prefix.Substring(1);
}
ListObjectsResponse listResponse = this._s3Client.ListObjects(listRequest);
List<S3Object> objs = new List<S3Object>();
foreach (S3Object s3o in listResponse.S3Objects)
{
DateTime lastModified = DateTime.Parse(s3o.LastModified);
if (this._request.IsSetModifiedSinceDate() && this._request.ModifiedSinceDate < lastModified)
continue;
if (this._request.IsSetUnmodifiedSinceDate() && lastModified < this._request.UnmodifiedSinceDate)
continue;
objs.Add(s3o);
}
this._totalNumberOfFilesToDownload = objs.Count;
foreach (S3Object s3o in objs)
{
this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length);
TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest();
downloadRequest.BucketName = this._request.BucketName;
downloadRequest.Key = s3o.Key;
downloadRequest.FilePath = Path.Combine(this._request.LocalDirectory, this._currentFile);
downloadRequest.Timeout = this._request.Timeout;
downloadRequest.WithSubscriber(new EventHandler<WriteObjectProgressArgs>(downloadedProgressEventCallback));
DownloadCommand command = new DownloadCommand(this._s3Client, downloadRequest);
command.Execute();
this._numberOfFilesDownloaded++;
}
}
public override async Task ExecuteAsync(CancellationToken cancellationToken)
{
ValidateRequest();
EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory));
ListObjectsRequest listRequest = ConstructListObjectRequest();
List <S3Object> objs = new List <S3Object>();
do
{
ListObjectsResponse listResponse = await this._s3Client.ListObjectsAsync(listRequest, cancellationToken).
ConfigureAwait(continueOnCapturedContext: false);
foreach (S3Object s3o in listResponse.S3Objects)
{
if (this._request.IsSetModifiedSinceDate() && s3o.LastModified <= this._request.ModifiedSinceDate)
{
continue;
}
if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified > this._request.UnmodifiedSinceDate)
{
continue;
}
this._totalBytes += s3o.Size;
objs.Add(s3o);
}
listRequest.Marker = listResponse.NextMarker;
} while (!string.IsNullOrEmpty(listRequest.Marker));
this._totalNumberOfFilesToDownload = objs.Count;
SemaphoreSlim asyncThrottler = null;
CancellationTokenSource internalCts = null;
try
{
asyncThrottler = DownloadFilesConcurrently ?
new SemaphoreSlim(this._config.ConcurrentServiceRequests) :
new SemaphoreSlim(1);
internalCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
var pendingTasks = new List <Task>();
foreach (S3Object s3o in objs)
{
if (s3o.Key.EndsWith("/", StringComparison.Ordinal))
{
continue;
}
await asyncThrottler.WaitAsync(cancellationToken).
ConfigureAwait(continueOnCapturedContext: false);
cancellationToken.ThrowIfCancellationRequested();
if (internalCts.IsCancellationRequested)
{
// Operation cancelled as one of the download requests failed with an exception,
// don't schedule any more download tasks.
// Don't throw an OperationCanceledException here as we want to process the
// responses and throw the original exception.
break;
}
// Valid for serial uploads when
// TransferUtilityDownloadDirectoryRequest.DownloadFilesConcurrently is set to false.
this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length);
var downloadRequest = ConstructTransferUtilityDownloadRequest(s3o, listRequest.Prefix.Length);
var command = new DownloadCommand(this._s3Client, downloadRequest);
var task = ExecuteCommandAsync(command, internalCts, asyncThrottler);
pendingTasks.Add(task);
}
await WhenAllOrFirstExceptionAsync(pendingTasks, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
}
finally
{
internalCts.Dispose();
asyncThrottler.Dispose();
}
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// If the key is not specified in the request parameter,
/// the file name will used as the key name.
/// </summary>
/// <param name="request">
/// Contains all the parameters required to download an Amazon S3 object.
/// </param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task DownloadAsync(TransferUtilityDownloadRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
var command = new DownloadCommand(this._s3Client, request);
return command.ExecuteAsync(cancellationToken);
}
public override void Execute()
{
if (!this._request.IsSetBucketName())
{
throw new InvalidOperationException("The bucketName Specified is null or empty!");
}
if (!this._request.IsSetS3Directory())
{
throw new InvalidOperationException("The S3Directory Specified is null or empty!");
}
if (!this._request.IsSetLocalDirectory())
{
throw new InvalidOperationException("The LocalDirectory Specified is null or empty!");
}
if (File.Exists(this._request.S3Directory))
{
throw new InvalidOperationException("A file already exists with the same name indicated by LocalDirectory!");
}
EnsureDirectoryExists(new DirectoryInfo(this._request.LocalDirectory));
ListObjectsRequest listRequest = new ListObjectsRequest();
listRequest.BucketName = this._request.BucketName;
listRequest.Prefix = this._request.S3Directory;
listRequest.Prefix = listRequest.Prefix.Replace('\\', '/');
if (!listRequest.Prefix.EndsWith("/", StringComparison.Ordinal))
listRequest.Prefix += "/";
if (listRequest.Prefix.StartsWith("/", StringComparison.Ordinal))
{
if (listRequest.Prefix.Length == 1)
listRequest.Prefix = "";
else
listRequest.Prefix = listRequest.Prefix.Substring(1);
}
List<S3Object> objs = new List<S3Object>();
do
{
ListObjectsResponse listResponse = this._s3Client.ListObjects(listRequest);
foreach (S3Object s3o in listResponse.S3Objects)
{
if (this._request.IsSetModifiedSinceDate() && this._request.ModifiedSinceDate < s3o.LastModified)
continue;
if (this._request.IsSetUnmodifiedSinceDate() && s3o.LastModified < this._request.UnmodifiedSinceDate)
continue;
objs.Add(s3o);
}
listRequest.Marker = listResponse.NextMarker;
} while (!string.IsNullOrEmpty(listRequest.Marker));
this._totalNumberOfFilesToDownload = objs.Count;
foreach (S3Object s3o in objs)
{
if (s3o.Key.EndsWith("/", StringComparison.Ordinal))
continue;
this._currentFile = s3o.Key.Substring(listRequest.Prefix.Length);
TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest();
downloadRequest.BucketName = this._request.BucketName;
downloadRequest.Key = s3o.Key;
downloadRequest.FilePath = Path.Combine(this._request.LocalDirectory, this._currentFile);
downloadRequest.WriteObjectProgressEvent += downloadedProgressEventCallback;
DownloadCommand command = new DownloadCommand(this._s3Client, downloadRequest);
command.Execute();
this._numberOfFilesDownloaded++;
}
}
