/// <summary>
/// Warning, if the book already exists in the location, this is going to delete it an over-write it. So it's up to the caller to check the sanity of that.
/// </summary>
/// <param name="storageKeyOfBookFolder"></param>
public string DownloadBook(string bucketName, string storageKeyOfBookFolder, string pathToDestinationParentDirectory,
IProgressDialog downloadProgress = null)
{
//review: should we instead save to a newly created folder so that we don't have to worry about the
//other folder existing already? Todo: add a test for that first.
// We need to download individual files to avoid downloading unwanted files (PDFs and thumbs.db to
// be specific). See https://silbloom.myjetbrains.com/youtrack/issue/BL-2312. So we need the list
// of items, not just the count.
var matching = GetMatchingItems(bucketName, storageKeyOfBookFolder);
var totalItems = CountDesiredFiles(matching);
if(totalItems == 0)
throw new DirectoryNotFoundException("The book we tried to download is no longer in the BloomLibrary");
Debug.Assert(matching.S3Objects[0].Key.StartsWith(storageKeyOfBookFolder + "/"));
// Get the top-level directory name of the book from the first object key.
var bookFolderName = matching.S3Objects[0].Key.Substring(storageKeyOfBookFolder.Length + 1);
while(bookFolderName.Contains("/"))
bookFolderName = Path.GetDirectoryName(bookFolderName);
// Amazon.S3 appears to truncate titles at 50 characters when building directory and filenames. This means
// that relative paths can be as long as 117 characters (2 * 50 + 2 for slashes + 15 for .BloomBookOrder).
// So our temporary folder must be no more than 140 characters (allow some margin) since paths can be a
// maximum of 260 characters in Windows. (More margin than that may be needed because there's no guarantee
// that image filenames are no longer than 65 characters.) See https://jira.sil.org/browse/BL-1160.
using(var tempDestination = new TemporaryFolder("BDS_" + Guid.NewGuid()))
{
var tempDirectory = Path.Combine(tempDestination.FolderPath, bookFolderName);
if(downloadProgress != null)
downloadProgress.Invoke((Action) (() => { downloadProgress.ProgressRangeMaximum = totalItems; }));
int booksDownloaded = 0;
using(var transferUtility = new TransferUtility(_amazonS3))
{
for(int i = 0; i < matching.S3Objects.Count; ++i)
{
var objKey = matching.S3Objects[i].Key;
if(AvoidThisFile(objKey))
continue;
// Removing the book's prefix from the object key, then using the remainder of the key
// in the filepath allows for nested subdirectories.
var filepath = objKey.Substring(storageKeyOfBookFolder.Length + 1);
// Download this file then bump progress.
var req = new TransferUtilityDownloadRequest()
{
BucketName = bucketName,
Key = objKey,
FilePath = Path.Combine(tempDestination.FolderPath, filepath)
};
transferUtility.Download(req);
++booksDownloaded;
if(downloadProgress != null)
downloadProgress.Invoke((Action) (() => { downloadProgress.Progress = booksDownloaded; }));
}
var destinationPath = Path.Combine(pathToDestinationParentDirectory, bookFolderName);
//clear out anything existing on our target
var didDelete = false;
if(Directory.Exists(destinationPath))
{
try
{
SIL.IO.RobustIO.DeleteDirectory(destinationPath, true);
didDelete = true;
}
catch(IOException)
{
// can't delete it...see if we can copy into it.
}
}
//if we're on the same volume, we can just move it. Else copy it.
// It's important that books appear as nearly complete as possible, because a file watcher will very soon add the new
// book to the list of downloaded books the user can make new ones from, once it appears in the target directory.
bool done = false;
if(didDelete && PathUtilities.PathsAreOnSameVolume(pathToDestinationParentDirectory, tempDirectory))
{
try
{
SIL.IO.RobustIO.MoveDirectory(tempDirectory, destinationPath);
done = true;
}
catch(IOException)
{
// If moving didn't work we'll just try copying
}
catch(UnauthorizedAccessException)
{
}
}
if(!done)
done = CopyDirectory(tempDirectory, destinationPath);
if(!done)
{
var msg = LocalizationManager.GetString("Download.CopyFailed",
"Bloom downloaded the book but had problems making it available in Bloom. Please restart your computer and try again. If you get this message again, please click the 'Details' button and report the problem to the Bloom developers");
// The exception doesn't add much useful information but it triggers a version of the dialog with a Details button
// that leads to the yellow box and an easy way to send the report.
ErrorReport.NotifyUserOfProblem(new ApplicationException("File Copy problem"), msg);
}
return destinationPath;
}
}
}
public void SimpleUpload()
{
var client = Client;
using (var tu = new Amazon.S3.Transfer.TransferUtility(client))
{
tu.Upload(fullPath, bucketName);
var response = client.GetObjectMetadata(new GetObjectMetadataRequest
{
BucketName = bucketName,
Key = testFile
});
Assert.IsTrue(response.ETag.Length > 0);
var downloadPath = fullPath + ".download";
var downloadRequest = new Amazon.S3.Transfer.TransferUtilityDownloadRequest
{
BucketName = bucketName,
Key = testFile,
FilePath = downloadPath
};
tu.Download(downloadRequest);
TestDownloadedFile(downloadPath);
// empty out file, except for 1 byte
File.WriteAllText(downloadPath, testContent.Substring(0, 1));
Assert.IsTrue(File.Exists(downloadPath));
tu.Download(downloadRequest);
TestDownloadedFile(downloadPath);
}
}
protected override void ExecuteS3Task()
{
using ( TransferUtility transferUtility = new Amazon.S3.Transfer.TransferUtility( this.AccessKey, this.SecretAccessKey ) ) {
TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest {
BucketName = this.BucketName,
Key = this.SourceFile,
FilePath = this.DestinationFile,
};
// uploadRequest.AddHeader("x-amz-acl", "public-read");
transferUtility.Download( downloadRequest );
}
}
public void SimpleUpload()
{
var client = Client;
using (var tu = new Amazon.S3.Transfer.TransferUtility(client))
{
tu.Upload(testFilePath, testBucketName);
var response = WaitUtils.WaitForComplete(
() =>
{
return(client.GetObjectMetadataAsync(new GetObjectMetadataRequest
{
BucketName = testBucketName,
Key = TEST_FILENAME
}).Result);
});
Assert.True(response.ETag.Length > 0);
var downloadPath = testFilePath + ".download";
var downloadRequest = new Amazon.S3.Transfer.TransferUtilityDownloadRequest
{
BucketName = testBucketName,
Key = TEST_FILENAME,
FilePath = downloadPath
};
tu.Download(downloadRequest);
TestDownloadedFile(downloadPath);
// empty out file, except for 1 byte
File.WriteAllText(downloadPath, TEST_CONTENT.Substring(0, 1));
Assert.True(File.Exists(downloadPath));
tu.Download(downloadRequest);
TestDownloadedFile(downloadPath);
}
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// If the key is not specified in the request parameter,
/// the file name will used as the key name.
/// </summary>
/// <param name="request">
/// Contains all the parameters required to download an Amazon S3 object.
/// </param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task DownloadAsync(TransferUtilityDownloadRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
var command = new DownloadCommand(this._s3Client, request);
return(command.ExecuteAsync(cancellationToken));
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// If the key is not specified in the request parameter,
/// the file name will used as the key name.
/// </summary>
/// <param name="request">
/// Contains all the parameters required to download an Amazon S3 object.
/// </param>
public void Download(TransferUtilityDownloadRequest request)
{
DownloadHelper(request);
}
/// <summary>
/// Initiates the asynchronous execution of the Download operation.
/// <seealso cref="M:Amazon.S3.Transfer.TransferUtility.Download"/>
/// </summary>
/// <param name="filePath">
/// The file path where the content from Amazon S3 will be written to.
/// </param>
/// <param name="bucketName">
/// The name of the bucket containing the Amazon S3 object to download.
/// </param>
/// <param name="key">
/// The key under which the Amazon S3 object is stored.
/// </param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback procedure using the AsyncState property.</param>
/// <exception cref="T:System.ArgumentNullException"></exception>
/// <exception cref="T:System.Net.WebException"></exception>
/// <exception cref="T:Amazon.S3.AmazonS3Exception"></exception>
/// <returns>An IAsyncResult that can be used to poll, or wait for results, or both.
/// This values is also needed when invoking EndDownload.</returns>
public IAsyncResult BeginDownload(string filePath, string bucketName, string key, AsyncCallback callback, object state)
{
TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest()
{
BucketName = bucketName,
Key = key,
FilePath = filePath
};
return BeginDownload(request, callback, state);
}
private TransferUtilityDownloadRequest CreateDownloadRequest(DownloadSettings settings)
{
TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest();
request.BucketName = settings.BucketName;
request.ServerSideEncryptionCustomerProvidedKey = settings.EncryptionKey;
request.ServerSideEncryptionCustomerProvidedKeyMD5 = settings.EncryptionKeyMD5;
request.ServerSideEncryptionCustomerMethod = settings.EncryptionMethod;
if (!String.IsNullOrEmpty(settings.EncryptionKey))
{
request.ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256;
}
request.ModifiedSinceDate = settings.ModifiedDate;
return request;
}
public Task DownloadFile(IProgress<TaskProgressReport> taskProgress, String fileMd5)
{
_progress = taskProgress;
var task = new Task((t) =>
{
if (_token.IsCancellationRequested)
{
_token.ThrowIfCancellationRequested();
}
try
{
var fileDownloadUtilityRequest = new TransferUtilityDownloadRequest
{
BucketName = _bucketName,
FilePath = _sourceFilePath,
Key = _name,
};
fileDownloadUtilityRequest.WriteObjectProgressEvent += DownloadProgressEvent;
OnStateChanged(new StateChangeEventArg(FileTransferState.InProgress));
_fileTransferUtility.DownloadAsync(fileDownloadUtilityRequest, _token).Wait(_token);
VerifyDownload(fileMd5);
}
catch (AmazonS3Exception e)
{
_logger.Error(e.Message, e);
//throw;
}
}, _token);
return task;
}
/// <summary>
/// Initiates the asynchronous execution of the Download operation.
/// <seealso cref="M:Amazon.S3.Transfer.TransferUtility.Download"/>
/// </summary>
/// <param name="request">
/// Contains all the parameters used to download an Amazon S3 object.
/// </param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback procedure using the AsyncState property.</param>
/// <exception cref="T:System.ArgumentNullException"></exception>
/// <exception cref="T:System.Net.WebException"></exception>
/// <exception cref="T:Amazon.S3.AmazonS3Exception"></exception>
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both;
/// this value is also needed when invoking EndDownload.</returns>
public IAsyncResult BeginDownload(TransferUtilityDownloadRequest request, AsyncCallback callback, object state)
{
BaseCommand command = new DownloadCommand(this._s3Client, request);
return beginOperation(command, callback, state);
}
/// <summary>
/// Downloads the latest backup set from the Aws service.
/// </summary>
/// <param name="compressor">The compresor to use when decompressing the downloaded file.</param>
/// <returns>The path to the downloaded and decompressed backup file.</returns>
public string DownloadBackup(ICompressor compressor)
{
S3Object latest = this.GetLatestBackupItem();
string path = latest.Key;
if (!String.IsNullOrEmpty(this.Target.AwsPrefix) && path.StartsWith(this.Target.AwsPrefix, StringComparison.OrdinalIgnoreCase))
{
path = path.Substring(this.Target.AwsPrefix.Length);
if (path.StartsWith("/", StringComparison.Ordinal))
{
path = path.Substring(1);
}
}
path = Path.Combine(TempDir, path);
string fileName = Path.GetFileName(path);
if (File.Exists(path))
{
File.Delete(path);
}
TransferInfo info = new TransferInfo()
{
BytesTransferred = 0,
FileName = fileName,
FileSize = 0
};
using (TransferUtility transfer = new TransferUtility(S3Client))
{
TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest()
.WithBucketName(AwsConfig.BucketName)
.WithFilePath(path)
.WithKey(latest.Key);
request.WriteObjectProgressEvent += (sender, e) =>
{
info.BytesTransferred = e.TransferredBytes;
info.FileSize = e.TotalBytes;
this.Fire(this.TransferProgress, new TransferInfo(info));
};
this.Fire(this.TransferStart, new TransferInfo(info));
transfer.Download(request);
}
this.Fire(this.TransferComplete, new TransferInfo(info));
this.Fire(this.DecompressStart);
string decompressedPath = compressor.Decompress(path);
this.Fire(this.DecompressComplete);
File.Delete(path);
return decompressedPath;
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// If the key is not specified in the request parameter,
/// the file name will used as the key name.
/// </summary>
/// <param name="request">
/// Contains all the parameters required to download an Amazon S3 object.
/// </param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task DownloadAsync(TransferUtilityDownloadRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
var command = new DownloadCommand(this._s3Client, request);
return command.ExecuteAsync(cancellationToken);
}
private void DownloadHelper(string filePath, string bucketName, string key)
{
TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest()
{
BucketName = bucketName,
Key = key,
FilePath = filePath
};
DownloadHelper(request);
}
/// <summary>
/// Initiates the asynchronous execution of the Download operation.
/// <seealso cref="Amazon.S3.IAmazonS3.AbortMultipartUpload"/>
/// </summary>
/// <param name="request">
/// Contains all the parameters required to download an Amazon S3 object.
/// </param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task DownloadAsync(TransferUtilityDownloadRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
return(ExecuteAsync(() => DownloadHelper(request), cancellationToken));
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// If the key is not specified in the request parameter,
/// the file name will used as the key name.
/// </summary>
/// <param name="request">
/// Contains all the parameters required to download an Amazon S3 object.
/// </param>
public void Download(TransferUtilityDownloadRequest request)
{
BaseCommand command = new DownloadCommand(this._s3Client, request);
command.Execute();
}
/// <summary>
/// Initiates the asynchronous execution of the Download operation.
/// <seealso cref="M:Amazon.S3.Transfer.TransferUtility.Download"/>
/// </summary>
/// <param name="request">
/// Contains all the parameters required to download an Amazon S3 object.
/// </param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback procedure using the AsyncState property.</param>
/// <exception cref="T:System.ArgumentNullException"></exception>
/// <exception cref="T:System.Net.WebException"></exception>
/// <exception cref="T:Amazon.S3.AmazonS3Exception"></exception>
/// <returns>An IAsyncResult that can be used to poll, or wait for results, or both.
/// This values is also needed when invoking EndDownload.</returns>
public IAsyncResult BeginDownload(TransferUtilityDownloadRequest request, AsyncCallback callback, object state)
{
BaseCommand command = new DownloadCommand(this._s3Client, request);
return(beginOperation(command, callback, state));
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// If the key is not specified in the request parameter,
/// the file name will used as the key name.
/// </summary>
/// <param name="request">
/// Contains all the parameters required to download an Amazon S3 object.
/// </param>
public void Download(TransferUtilityDownloadRequest request)
{
DownloadHelper(request);
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// If the key is not specified in the request parameter,
/// the file name will be assumed.
/// </summary>
/// <param name="request">
/// Contains all the parameters used to download an Amazon S3 object.
/// </param>
public void Download(TransferUtilityDownloadRequest request)
{
BaseCommand command = new DownloadCommand(this._s3Client, request);
command.Execute();
}
/// <summary>
/// Initiates the asynchronous execution of the Download operation.
/// <seealso cref="M:Amazon.S3.Transfer.TransferUtility.Download"/>
/// </summary>
/// <param name="filePath">
/// The file path where the content from Amazon S3 will be written to.
/// </param>
/// <param name="bucketName">
/// The name of the bucket containing the Amazon S3 object to download.
/// </param>
/// <param name="key">
/// The key under which the Amazon S3 object is stored.
/// </param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback procedure using the AsyncState property.</param>
/// <exception cref="T:System.ArgumentNullException"></exception>
/// <exception cref="T:System.Net.WebException"></exception>
/// <exception cref="T:Amazon.S3.AmazonS3Exception"></exception>
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both;
/// this value is also needed when invoking EndDownload.</returns>
public IAsyncResult BeginDownload(string filePath, string bucketName, string key, AsyncCallback callback, object state)
{
TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest()
.WithBucketName(bucketName)
.WithKey(key)
.WithFilePath(filePath);
return BeginDownload(request, callback, state);
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// If the key is not specified in the request parameter,
/// the file name will be assumed.
/// </summary>
/// <param name="request">
/// Contains all the parameters used to download an Amazon S3 object.
/// </param>
public void Download(TransferUtilityDownloadRequest request)
{
if (!request.IsSetBucketName())
{
throw new ArgumentNullException("bucketName", "The bucketName Specified is null or empty!");
}
if (!request.IsSetFilePath())
{
throw new ArgumentNullException("filePath", "The filepath Specified is null or empty!");
}
if (!request.IsSetKey())
{
throw new ArgumentNullException("key", "The Key Specified is null or empty!");
}
GetObjectRequest getRequest = convertToGetObjectRequest(request);
GetObjectResponse response = this._s3Client.GetObject(getRequest);
response.WriteObjectProgressEvent += request.EventHandler;
response.WriteResponseStreamToFile(request.FilePath);
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// The object key is obtained from the file's file name.
/// </summary>
/// <param name="filePath">
/// The file path where the content from Amazon S3 will be written to.
/// </param>
/// <param name="bucketName">
/// The name of the bucket containing the Amazon S3 object to download.
/// </param>
/// <param name="key">
/// The key under which the Amazon S3 object is stored.
/// </param>
public void Download(string filePath, string bucketName, string key)
{
TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest()
.WithBucketName(bucketName)
.WithKey(key)
.WithFilePath(filePath);
Download(request);
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// If the key is not specified in the request parameter,
/// the file name will used as the key name.
/// </summary>
/// <param name="request">
/// Contains all the parameters required to download an Amazon S3 object.
/// </param>
public void Download(TransferUtilityDownloadRequest request)
{
try
{
DownloadAsync(request).Wait();
}
catch (AggregateException e)
{
ExceptionDispatchInfo.Capture(e.InnerException).Throw();
}
}
/// <summary>
/// Downloads the content from Amazon S3 and writes it to the specified file.
/// The object key is derived from the file name.
/// </summary>
/// <param name="filePath">
/// The file path where the content from Amazon S3 will be written to.
/// </param>
/// <param name="bucketName">
/// The name of the bucket containing the Amazon S3 object to download.
/// </param>
/// <param name="key">
/// The key under which the Amazon S3 object is stored.
/// </param>
public void Download(string filePath, string bucketName, string key)
{
TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest
{
BucketName = bucketName,
Key = key,
FilePath = filePath
};
Download(request);
}
/// <summary>
/// Transfers a file from S3. This method uses the "TransferUtility" class in order to download the file from S3
/// </summary>
public bool FileDownload(string bucketname, string dataname, string filepath)
{
// Reset error info
ClearErrorInfo();
// Load data
try
{
TransferUtilityDownloadRequest fileTransferUtilityRequest = new TransferUtilityDownloadRequest
{
BucketName = bucketname,
FilePath = filepath,
Key = dataname,
};
fileTransferUtility.Download(fileTransferUtilityRequest);
}
catch (Exception ex)
{
ErrorCode = e_Exception;
ErrorMessage = ex.Message + "::" + ex.InnerException;
}
return ErrorCode == 0;
}
public async Task DownloadFile(IStorageFile storageFile, string key, CancellationToken cancellationToken)
{
BasicAWSCredentials credentials = new BasicAWSCredentials("AKIAJTADDHY7T7GZXX5Q", "n4xV5B25mt7e6br84H2G9SXBx8eDYTQJgCxoaF49");
var s3Client = new AmazonS3Client(credentials, RegionEndpoint.USEast1);
//files = await Windows.Storage.DownloadsFolder.CreateFileAsync();
using (var transferUtility = new TransferUtility(s3Client))
{
var downloadRequest = new TransferUtilityDownloadRequest
{
BucketName = ExistingBucketName,
Key = key,
StorageFile = storageFile
};
downloadRequest.WriteObjectProgressEvent += OnWriteObjectProgressEvent;
await transferUtility.DownloadAsync(downloadRequest, cancellationToken);
}
}
public async Task DownloadFile(IStorageFile storageFile, string bucket, string key, AWSCredentials credentials, CancellationToken cancellationToken)
{
var s3Client = new AmazonS3Client(credentials, RegionEndpoint.USEast1);
using (var transferUtility = new TransferUtility(s3Client))
{
var downloadRequest = new TransferUtilityDownloadRequest
{
BucketName = bucket,
Key = key,
StorageFile = storageFile
};
downloadRequest.WriteObjectProgressEvent += OnWriteObjectProgressEvent;
await transferUtility.DownloadAsync(downloadRequest, cancellationToken);
}
}
public async Task DownloadTestFile()
{
var token = await S3Service.GetS3Token("upload");
string filePath = "c:/temp/download/test.jpg";
string awsAccessKeyId = token.accessKeyId;
string awsSecretAccessKey = token.secretAccessKey;
string awsSessionToken = token.sessionToken;
string existingBucketName = token.bucket;
string keyName = string.Format("{0}/{1}", token.path, Path.GetFileName(filePath));
var client = new AmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsSessionToken, RegionEndpoint.APSoutheast2);
var fileTransferUtility = new TransferUtility(client);
var request = new TransferUtilityDownloadRequest()
{
BucketName = existingBucketName,
FilePath = filePath,
Key = keyName,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = token.uploadPassword,
};
fileTransferUtility.Download(request);
Console.WriteLine("download 1 completed");
}
