// Download file from bucket to local location
public void Downloadfile(String bucket, String file, String storePath)
{
try
{
TransferUtility fileTransferUtility = new
TransferUtility(new AmazonS3Client(Amazon.RegionEndpoint.EUWest1));
// 2. Specify object key name explicitly.
fileTransferUtility.Download(storePath,
bucket, file);
Console.WriteLine(String.Format("Succesfully downloaded file: {0} from bucket: {1} to location: {2}", file, bucket, storePath));
}
catch (AmazonS3Exception s3Exception)
{
Console.WriteLine(s3Exception.Message,
s3Exception.InnerException);
}
}
/// <summary>
/// Warning, if the book already exists in the location, this is going to delete it an over-write it. So it's up to the caller to check the sanity of that.
/// </summary>
/// <param name="storageKeyOfBookFolder"></param>
public string DownloadBook(string bucketName, string storageKeyOfBookFolder, string pathToDestinationParentDirectory,
IProgressDialog downloadProgress = null)
{
//review: should we instead save to a newly created folder so that we don't have to worry about the
//other folder existing already? Todo: add a test for that first.
// We need to download individual files to avoid downloading unwanted files (PDFs and thumbs.db to
// be specific). See https://silbloom.myjetbrains.com/youtrack/issue/BL-2312. So we need the list
// of items, not just the count.
var matching = GetMatchingItems(bucketName, storageKeyOfBookFolder);
var totalItems = CountDesiredFiles(matching);
if(totalItems == 0)
throw new DirectoryNotFoundException("The book we tried to download is no longer in the BloomLibrary");
Debug.Assert(matching.S3Objects[0].Key.StartsWith(storageKeyOfBookFolder + "/"));
// Get the top-level directory name of the book from the first object key.
var bookFolderName = matching.S3Objects[0].Key.Substring(storageKeyOfBookFolder.Length + 1);
while(bookFolderName.Contains("/"))
bookFolderName = Path.GetDirectoryName(bookFolderName);
// Amazon.S3 appears to truncate titles at 50 characters when building directory and filenames. This means
// that relative paths can be as long as 117 characters (2 * 50 + 2 for slashes + 15 for .BloomBookOrder).
// So our temporary folder must be no more than 140 characters (allow some margin) since paths can be a
// maximum of 260 characters in Windows. (More margin than that may be needed because there's no guarantee
// that image filenames are no longer than 65 characters.) See https://jira.sil.org/browse/BL-1160.
using(var tempDestination = new TemporaryFolder("BDS_" + Guid.NewGuid()))
{
var tempDirectory = Path.Combine(tempDestination.FolderPath, bookFolderName);
if(downloadProgress != null)
downloadProgress.Invoke((Action) (() => { downloadProgress.ProgressRangeMaximum = totalItems; }));
int booksDownloaded = 0;
using(var transferUtility = new TransferUtility(_amazonS3))
{
for(int i = 0; i < matching.S3Objects.Count; ++i)
{
var objKey = matching.S3Objects[i].Key;
if(AvoidThisFile(objKey))
continue;
// Removing the book's prefix from the object key, then using the remainder of the key
// in the filepath allows for nested subdirectories.
var filepath = objKey.Substring(storageKeyOfBookFolder.Length + 1);
// Download this file then bump progress.
var req = new TransferUtilityDownloadRequest()
{
BucketName = bucketName,
Key = objKey,
FilePath = Path.Combine(tempDestination.FolderPath, filepath)
};
transferUtility.Download(req);
++booksDownloaded;
if(downloadProgress != null)
downloadProgress.Invoke((Action) (() => { downloadProgress.Progress = booksDownloaded; }));
}
var destinationPath = Path.Combine(pathToDestinationParentDirectory, bookFolderName);
//clear out anything existing on our target
var didDelete = false;
if(Directory.Exists(destinationPath))
{
try
{
SIL.IO.RobustIO.DeleteDirectory(destinationPath, true);
didDelete = true;
}
catch(IOException)
{
// can't delete it...see if we can copy into it.
}
}
//if we're on the same volume, we can just move it. Else copy it.
// It's important that books appear as nearly complete as possible, because a file watcher will very soon add the new
// book to the list of downloaded books the user can make new ones from, once it appears in the target directory.
bool done = false;
if(didDelete && PathUtilities.PathsAreOnSameVolume(pathToDestinationParentDirectory, tempDirectory))
{
try
{
SIL.IO.RobustIO.MoveDirectory(tempDirectory, destinationPath);
done = true;
}
catch(IOException)
{
// If moving didn't work we'll just try copying
}
catch(UnauthorizedAccessException)
{
}
}
if(!done)
done = CopyDirectory(tempDirectory, destinationPath);
if(!done)
{
var msg = LocalizationManager.GetString("Download.CopyFailed",
"Bloom downloaded the book but had problems making it available in Bloom. Please restart your computer and try again. If you get this message again, please click the 'Details' button and report the problem to the Bloom developers");
// The exception doesn't add much useful information but it triggers a version of the dialog with a Details button
// that leads to the yellow box and an easy way to send the report.
ErrorReport.NotifyUserOfProblem(new ApplicationException("File Copy problem"), msg);
}
return destinationPath;
}
}
}
public void SimpleUpload()
{
var client = Client;
using (var tu = new Amazon.S3.Transfer.TransferUtility(client))
{
tu.Upload(fullPath, bucketName);
var response = client.GetObjectMetadata(new GetObjectMetadataRequest
{
BucketName = bucketName,
Key = testFile
});
Assert.IsTrue(response.ETag.Length > 0);
var downloadPath = fullPath + ".download";
var downloadRequest = new Amazon.S3.Transfer.TransferUtilityDownloadRequest
{
BucketName = bucketName,
Key = testFile,
FilePath = downloadPath
};
tu.Download(downloadRequest);
TestDownloadedFile(downloadPath);
// empty out file, except for 1 byte
File.WriteAllText(downloadPath, testContent.Substring(0, 1));
Assert.IsTrue(File.Exists(downloadPath));
tu.Download(downloadRequest);
TestDownloadedFile(downloadPath);
}
}
protected override void ExecuteS3Task()
{
using (TransferUtility transferUtility = new Amazon.S3.Transfer.TransferUtility(this.AccessKey, this.SecretAccessKey)) {
TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest {
BucketName = this.BucketName,
Key = this.SourceFile,
FilePath = this.DestinationFile,
};
// uploadRequest.AddHeader("x-amz-acl", "public-read");
transferUtility.Download(downloadRequest);
}
}
protected override void ExecuteS3Task()
{
using ( TransferUtility transferUtility = new Amazon.S3.Transfer.TransferUtility( this.AccessKey, this.SecretAccessKey ) ) {
TransferUtilityDownloadRequest downloadRequest = new TransferUtilityDownloadRequest {
BucketName = this.BucketName,
Key = this.SourceFile,
FilePath = this.DestinationFile,
};
// uploadRequest.AddHeader("x-amz-acl", "public-read");
transferUtility.Download( downloadRequest );
}
}
public void SimpleUpload()
{
var client = Client;
using (var tu = new Amazon.S3.Transfer.TransferUtility(client))
{
tu.Upload(testFilePath, testBucketName);
var response = WaitUtils.WaitForComplete(
() =>
{
return(client.GetObjectMetadataAsync(new GetObjectMetadataRequest
{
BucketName = testBucketName,
Key = TEST_FILENAME
}).Result);
});
Assert.True(response.ETag.Length > 0);
var downloadPath = testFilePath + ".download";
var downloadRequest = new Amazon.S3.Transfer.TransferUtilityDownloadRequest
{
BucketName = testBucketName,
Key = TEST_FILENAME,
FilePath = downloadPath
};
tu.Download(downloadRequest);
TestDownloadedFile(downloadPath);
// empty out file, except for 1 byte
File.WriteAllText(downloadPath, TEST_CONTENT.Substring(0, 1));
Assert.True(File.Exists(downloadPath));
tu.Download(downloadRequest);
TestDownloadedFile(downloadPath);
}
}
/// <summary>
/// Downloads the latest backup set from the Aws service.
/// </summary>
/// <param name="compressor">The compresor to use when decompressing the downloaded file.</param>
/// <returns>The path to the downloaded and decompressed backup file.</returns>
public string DownloadBackup(ICompressor compressor)
{
S3Object latest = this.GetLatestBackupItem();
string path = latest.Key;
if (!String.IsNullOrEmpty(this.Target.AwsPrefix) && path.StartsWith(this.Target.AwsPrefix, StringComparison.OrdinalIgnoreCase))
{
path = path.Substring(this.Target.AwsPrefix.Length);
if (path.StartsWith("/", StringComparison.Ordinal))
{
path = path.Substring(1);
}
}
path = Path.Combine(TempDir, path);
string fileName = Path.GetFileName(path);
if (File.Exists(path))
{
File.Delete(path);
}
TransferInfo info = new TransferInfo()
{
BytesTransferred = 0,
FileName = fileName,
FileSize = 0
};
using (TransferUtility transfer = new TransferUtility(S3Client))
{
TransferUtilityDownloadRequest request = new TransferUtilityDownloadRequest()
.WithBucketName(AwsConfig.BucketName)
.WithFilePath(path)
.WithKey(latest.Key);
request.WriteObjectProgressEvent += (sender, e) =>
{
info.BytesTransferred = e.TransferredBytes;
info.FileSize = e.TotalBytes;
this.Fire(this.TransferProgress, new TransferInfo(info));
};
this.Fire(this.TransferStart, new TransferInfo(info));
transfer.Download(request);
}
this.Fire(this.TransferComplete, new TransferInfo(info));
this.Fire(this.DecompressStart);
string decompressedPath = compressor.Decompress(path);
this.Fire(this.DecompressComplete);
File.Delete(path);
return decompressedPath;
}
public async Task DownloadTestFile()
{
var token = await S3Service.GetS3Token("upload");
string filePath = "c:/temp/download/test.jpg";
string awsAccessKeyId = token.accessKeyId;
string awsSecretAccessKey = token.secretAccessKey;
string awsSessionToken = token.sessionToken;
string existingBucketName = token.bucket;
string keyName = string.Format("{0}/{1}", token.path, Path.GetFileName(filePath));
var client = new AmazonS3Client(awsAccessKeyId, awsSecretAccessKey, awsSessionToken, RegionEndpoint.APSoutheast2);
var fileTransferUtility = new TransferUtility(client);
var request = new TransferUtilityDownloadRequest()
{
BucketName = existingBucketName,
FilePath = filePath,
Key = keyName,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = token.uploadPassword,
};
fileTransferUtility.Download(request);
Console.WriteLine("download 1 completed");
}