public static GlacierResult Upload(string vaultName, string archivePath)
{
GlacierResult glacierResult = new GlacierResult();
try
{
var manager = new ArchiveTransferManager(Amazon.RegionEndpoint.APNortheast1);
string description = string.Format("{0} upload to AmazonGlacier", DateTime.Now.ToString());
//UploadResult result = manager.Upload(vaultName, description, archivePath);
//glacierResult.UploadResult = result;
glacierResult.UploadResult = new UploadResult(System.Guid.NewGuid().ToString());
}
catch (AmazonGlacierException e)
{
glacierResult.GlacierException = e;
}
catch (AmazonServiceException e)
{
glacierResult.AWSException = e;
}
catch (Exception e)
{
glacierResult.Exception = e;
}
return glacierResult;
}
public void runDownload()
{
try
{
//var manager = new ArchiveTransferManager(Amazon.RegionEndpoint.USWest2);
//var options = new DownloadOptions();
//options.StreamTransferProgress += ArchiveDownloadHighLevel.progress;
//// Download an archive.
//Console.WriteLine("Intiating the archive retrieval job and then polling SQS queue for the archive to be available.");
//Console.WriteLine("This polling takes about 4 hours. Once the archive is available, downloading will begin.");
//manager.Download(vaultName, archiveId, downloadFilePath, options);
//Console.WriteLine("To continue, press Enter");
//Console.ReadKey();
var manager = new ArchiveTransferManager(Amazon.RegionEndpoint.USEast1);
var options = new DownloadOptions();
options.StreamTransferProgress += DownloadFile.progress;
Console.WriteLine("Intiating the archive retrieval job and then polling SQS queue for the archive to be available.");
Console.WriteLine("This polling takes about 4 hours. Once the archive is available, downloading will begin.");
manager.Download(vaultName, archiveToDownload, downloadFilePath, options);
manager.Download(vaultName, archiveToDownload2, downloadFilePath2, options);
Console.WriteLine("To continue, press Enter");
Console.ReadKey();
}
catch (AmazonGlacierException e) { Console.WriteLine(e.Message); }
catch (AmazonServiceException e) { Console.WriteLine(e.Message); }
catch (Exception e) { Console.WriteLine(e.Message); }
Console.WriteLine("To continue, press Enter");
Console.ReadKey();
}
public void StartSession()
{
StopSession();
disposed = false;
Client = new AmazonGlacierClient(Mapping.AccessKey, Mapping.SecretKey, Mapping.Endpoint);
Manager = new ArchiveTransferManager(Client);
}
/*
* this appends a filename and file key pair to location specified in 'CopyArchiveKeys'
*/
public void runUpload()
{
CopyArchiveKeys cak = new CopyArchiveKeys(@"C:\Code\VisualStudio\Projects\AwsBackup2\KeyCopy\");
try{
var manager = new ArchiveTransferManager(Amazon.RegionEndpoint.USEast1);
string archiveID = manager.Upload(vaultName, "getting started test", archiveToUpload).ArchiveId;
Console.WriteLine("Archive ID (copy and save for next step): {0}", archiveID);
cak.copyThis(archiveToUpload+" "+"-"+" "+archiveID);
//cak.copyThis("test");
//Console.ReadKey();
}
catch (AmazonGlacierException e) { Console.WriteLine(e.Message);}
catch (AmazonServiceException e) { Console.WriteLine(e.Message);}
catch (Exception e) { Console.WriteLine(e.Message);}
Console.WriteLine("To continue, press Enter");
Console.ReadKey();
}
static void Main(string[] args)
{
if (CheckRequiredFields())
{
using (manager = new ArchiveTransferManager(RegionEndpoint.USWest2))
{
try
{
// Creates a new Vault
Console.WriteLine("Create Vault");
manager.CreateVault(vaultName);
// Uploads the specified file to Glacier.
Console.WriteLine("Upload a Archive");
var uploadResult = manager.Upload(vaultName, "Archive Description", filePath);
archiveId = uploadResult.ArchiveId;
Console.WriteLine("Upload successful. Archive Id : {0} Checksum : {1}",
uploadResult.ArchiveId, uploadResult.Checksum);
// Downloads the file from Glacier
// This operation can take a long time to complete.
// The ArchiveTransferManager.Download() method creates an Amazon SNS topic,
// and an Amazon SQS queue that is subscribed to that topic.
// It then initiates the archive retrieval job and polls the queue for the
// archive to be available. This polling takes about 4 hours.
// Once the archive is available, download will begin.
Console.WriteLine("Download the Archive");
var options = new DownloadOptions();
options.StreamTransferProgress += OnProgress;
manager.Download(vaultName, archiveId, downloadFilePath, options);
Console.WriteLine("Delete the Archive");
manager.DeleteArchive(vaultName, archiveId);
}
catch (AmazonGlacierException e)
{
Console.WriteLine(e.Message);
}
catch (AmazonServiceException e)
{
Console.WriteLine(e.Message);
}
}
}
}
private void UploadToGlacier(string backupPath, PeriodicBackupSetup localBackupConfigs)
{
var awsRegion = RegionEndpoint.GetBySystemName(localBackupConfigs.AwsRegionEndpoint) ?? RegionEndpoint.USEast1;
var manager = new ArchiveTransferManager(awsAccessKey, awsSecretKey, awsRegion);
var archiveId = manager.Upload(localBackupConfigs.GlacierVaultName, GetArchiveDescription(), backupPath).ArchiveId;
logger.Info(string.Format("Successfully uploaded backup {0} to Glacier, archive ID: {1}", Path.GetFileName(backupPath),
archiveId));
}
private void DoUpload(string backupPath, PeriodicBackupSetup backupConfigs)
{
var AWSRegion = RegionEndpoint.GetBySystemName(backupConfigs.AwsRegionEndpoint) ?? RegionEndpoint.USEast1;
var desc = string.Format("Raven.Database.Backup {0} {1}", Database.Name,
DateTimeOffset.UtcNow.ToString("u"));
if (!string.IsNullOrWhiteSpace(backupConfigs.GlacierVaultName))
{
var manager = new ArchiveTransferManager(awsAccessKey, awsSecretKey, AWSRegion);
var archiveId = manager.Upload(backupConfigs.GlacierVaultName, desc, backupPath).ArchiveId;
logger.Info(string.Format("Successfully uploaded backup {0} to Glacier, archive ID: {1}", Path.GetFileName(backupPath),
archiveId));
return;
}
if (!string.IsNullOrWhiteSpace(backupConfigs.S3BucketName))
{
var client = new Amazon.S3.AmazonS3Client(awsAccessKey, awsSecretKey, AWSRegion);
using (var fileStream = File.OpenRead(backupPath))
{
var key = Path.GetFileName(backupPath);
var request = new PutObjectRequest();
request.WithMetaData("Description", desc);
request.WithInputStream(fileStream);
request.WithBucketName(backupConfigs.S3BucketName);
request.WithKey(key);
using (S3Response _ = client.PutObject(request))
{
logger.Info(string.Format("Successfully uploaded backup {0} to S3 bucket {1}, with key {2}",
Path.GetFileName(backupPath), backupConfigs.S3BucketName, key));
return;
}
}
}
}
public string Upload(string archiveFilename, string archiveDescription = "")
{
var manager = new ArchiveTransferManager(RegionEndpoint);
return manager.Upload(Name, archiveDescription, archiveFilename).ArchiveId;
}
private Exception uploadArchive(ArchiveTransferManager manager, Archive arch, string basePath)
{
int rep = 0;
Exception error = null;
isDirty = true;
do
{
if (rep > 0)
{
Thread.Sleep(2000);
}
try
{
DateTime start = DateTime.Now;
arch.upload(manager, VaultName, basePath);
UploadTime += DateTime.Now.Subtract(start);
UploadBytes += arch.FileLength;
if (!inventoryByPath.ContainsKey(arch.RelativePath))
{
inventoryByPath[arch.RelativePath] = new List<Archive>();
}
inventoryByPath[arch.RelativePath].Add(arch);
inventoryById[arch.ArchiveId] = arch;
toUpload.Remove(arch);
}
catch(Exception ex)
{
error = ex;
}
}
while (++rep < 4 && error != null);
return error;
}
private Exception deleteArchive(ArchiveTransferManager manager, KeyValuePair<string, List<Archive>> entryInThis, Archive archive)
{
int rep = 0;
Exception error = null;
isDirty = true;
do
{
if (rep > 0)
{
Thread.Sleep(2000);
}
try
{
DateTime start = DateTime.Now;
manager.DeleteArchive(VaultName, archive.ArchiveId);
DeleteTime += DateTime.Now.Subtract(start);
entryInThis.Value.Remove(archive);
inventoryById.Remove(archive.ArchiveId);
if (entryInThis.Value.Count == 0)
{
inventoryByPath.Remove(entryInThis.Key);
}
toDelete.Remove(archive);
}
catch(Exception ex)
{
error = ex;
}
}
while (++rep < 4 && error != null);
return error;
}
public void uploadNewAndChanged(ArchiveTransferManager manager, string basePath, Dictionary<string, FileDetail> localFiles, Func<bool> keepRunning)
{
Archive arch;
bool hasMatch;
toUpload.Clear();
foreach (FileDetail file in localFiles.Values)
{
if (!keepRunning())
{
return;
}
if (!inventoryByPath.ContainsKey(file.RelativePath))
{
arch = new Archive(file);
toUpload.Add(arch);
}
else
{
hasMatch = false;
foreach (Archive a in inventoryByPath[file.RelativePath])
{
if (a.FileLength == file.FileLength && a.LastModified == file.LastModified)
{
hasMatch = true;
}
}
if (!hasMatch)
{
arch = new Archive(file);
toUpload.Add(arch);
}
}
}
toUpload.Sort();
toUpload.Reverse();
Exception error;
for (int i = toUpload.Count - 1; i >= 0 ; i--)
{
if (!keepRunning())
{
return;
}
arch = toUpload[i];
if ((error = uploadArchive(manager, arch, basePath)) == null)
{
UploadCount++;
}
else
{
uploadErrors[arch] = error;
}
}
}
public bool reuploadArchive(ArchiveTransferManager manager, string archiveId, string basePath)
{
Archive arch = inventoryById.ContainsKey(archiveId) ? inventoryById[archiveId] : null;
if (arch != null)
{
arch = new Archive(arch);
toUpload.Add(arch);
return uploadArchive(manager, arch, basePath) == null;
}
return false;
}
public void deleteRemovedAndOldVersions(ArchiveTransferManager manager, Dictionary<string, FileDetail> localFiles, Func<bool> keepRunning)
{
Dictionary<Archive, KeyValuePair<string, List<Archive>>> deleted = new Dictionary<Archive, KeyValuePair<string, List<Archive>>>();
DateTime compare = DateTime.Now;
TimeSpan maxAge = TimeSpan.FromDays(120.0);
List<string> empty = new List<string>();
bool hasGoodEntry;
foreach (KeyValuePair<string, List<Archive>> archives in inventoryByPath)
{
if (!keepRunning())
{
return;
}
hasGoodEntry = false;
foreach (Archive archive in archives.Value)
{
if (!keepRunning())
{
return;
}
if (hasGoodEntry || !localFiles.ContainsKey(archive.RelativePath) ||
localFiles[archive.RelativePath].FileLength != archive.FileLength ||
localFiles[archive.RelativePath].LastModified != archive.LastModified)
{
archive.DeletionDate = archive.DeletionDate == null ? DateTime.Now : archive.DeletionDate;
if (compare.Subtract((DateTime)archive.DeletionDate) > maxAge)
{
deleted[archive] = archives;
}
}
else
{
hasGoodEntry = true;
}
}
}
toDelete = new List<Archive>(deleted.Keys);
toDelete.Sort();
toDelete.Reverse();
Archive arch;
Exception error;
for(int i = toDelete.Count - 1; i >= 0; i--)
{
if (!keepRunning())
{
return;
}
arch = toDelete[i];
if ((error = deleteArchive(manager, deleted[arch], arch)) == null)
{
DeleteCount++;
}
else
{
deleteErrors[arch] = error;
}
}
}
public bool deleteArchive(ArchiveTransferManager manager, string archiveId)
{
KeyValuePair<string, List<Archive>> entry = new KeyValuePair<string,List<Archive>>();
Archive archive = null;
foreach (KeyValuePair<string, List<Archive>> archives in inventoryByPath)
{
foreach (Archive arch in archives.Value)
{
if (arch.ArchiveId == archiveId)
{
archive = arch;
entry = archives;
break;
}
}
if (archive != null)
{
break;
}
}
return archive == null ? true : deleteArchive(manager, entry, archive) == null;
}
public FDGlacier(FDUserSettings settings, FDOperationLog oplog, string optype)
{
this.vaultName = settings.AWSGlacierVaultName;
log = oplog;
switch (settings.AWSRegion) {
case FDUserSettings.AWSRegionIndex.USWest1:
region = RegionEndpoint.USWest1;
break;
case FDUserSettings.AWSRegionIndex.USWest2:
region = RegionEndpoint.USWest2;
break;
case FDUserSettings.AWSRegionIndex.USEast1:
region = RegionEndpoint.USEast1;
break;
case FDUserSettings.AWSRegionIndex.EUWest1:
region = RegionEndpoint.EUWest1;
break;
case FDUserSettings.AWSRegionIndex.APNortheast1:
region = RegionEndpoint.APNortheast1;
break;
default:
region = RegionEndpoint.USEast1;
break;
}
//Instantiate the glacier config with our settins (for future move to AmazonGlacierClient)
glacierConfig = new AmazonGlacierConfig();
glacierConfig.RegionEndpoint = region;
//Instantiate AWS Credentials
awsCredentials = new BasicAWSCredentials(settings.AWSAccessKey, settings.AWSSecretKey);
//Instantiate the transfer manager with our settings
//TODO: Switch to glacier client so we can abort this damn thing
//glacierClient = new AmazonGlacierClient(appConfig["AWSAccessKey"], appConfig["AWSSecretKey"], region);
transferManager = new ArchiveTransferManager(awsCredentials, region);
upOptions = new UploadOptions();
downOptions = new DownloadOptions();
progress = 0;
upOptions.StreamTransferProgress = downOptions.StreamTransferProgress = this.onProgress;
OperationType = optype;
}
public static void Upload(string[] args)
{
string id = args[0];
string access = args[1];
string secret = args[2];
RegionEndpoint region = ParseRegion(args[3]);
string vault = args[4];
string path = args[5];
string desc = args[6];
var info = new FileInfo(path);
long size = 0;
if (info.Exists)
size = info.Length;
else
{
Console.Error.Write("The given path does not exist.\n");
Usage();
}
Console.Write("About to perform the following upload:\n\n" +
String.Format("{0,-16}{1}\n", "Path:", path) +
String.Format("{0,-16}{1:f6} GiB\n", "Size:",
(decimal)size / (1024m * 1024m * 1024m)) +
String.Format("{0,-16}\"{1}\"\n", "Description:", desc) +
String.Format("\n{0,-16}{1}\n", "Region:", region.DisplayName) +
String.Format("{0,-16}{1}\n", "Vault:", vault) +
String.Format("\n{0,-16}${1:f2}/month\n", "Storage cost:",
((decimal)size / (1024m * 1024m * 1024m) * 0.01m)) +
String.Format("{0,-16}${1:f2} (< 90 days)\n", "Deletion fee:",
((decimal)size / (1024m * 1024m * 1024m) * 0.03m)) +
String.Format("{0,-16}${1:f2}\n", "Retrieval cost:",
((decimal)size / (1024m * 1024m * 1024m) * 0.12m)) +
String.Format("\n{0,-16}{1}\n", "Account ID:", id) +
String.Format("{0,-16}{1}\n", "Access key:", access) +
String.Format("{0,-16}{1}\n", "Secret key:", secret) +
"\nARE YOU SURE YOU WANT TO PROCEED? [y/N] ");
bool proceed = false;
do
{
ConsoleKeyInfo key = Console.ReadKey(true);
switch (Char.ToLower(key.KeyChar))
{
case 'y':
Console.WriteLine(key.KeyChar);
proceed = true;
break;
case 'n':
Console.Write(key.KeyChar);
goto case '\n';
case '\n':
Console.WriteLine();
Console.Write("Upload aborted.\n");
Environment.Exit(0);
break;
}
}
while (!proceed);
Console.Write("\nUpload started at {0:G}.\n", DateTime.Now);
Console.CancelKeyPress += CtrlC;
var progress = new Progress();
var options = new UploadOptions();
options.AccountId = id;
options.StreamTransferProgress += progress.Update;
var creds = new BasicAWSCredentials(access, secret);
var manager = new ArchiveTransferManager(creds, region);
progress.Start();
/* not asynchronous */
UploadResult result = manager.Upload(vault, desc, path, options);
Console.Write("\n\nUpload complete.\n" +
String.Format("Archive ID: {0}\n", result.ArchiveId));
}
private bool handleMessage(QueueMessage message)
{
if(!KeepRunning())
{
return false;
}
if (active)
{
if ((message.Action == "InventoryRetrieval" || message.Action == "ArchiveRetrieval") && message.JobId != null)
{
completeJob(message.Properties, message.Action);
return true;
}
else if (message.Action == "ReuploadArchive" && message.ArchiveId != null)
{
using (ArchiveTransferManager manager = new ArchiveTransferManager(Context.Mapping.AccessKey, Context.Mapping.SecretKey, Context.Mapping.Endpoint))
{
Context.Vault.reuploadArchive(manager, message.ArchiveId, Context.Mapping.LocalFolder);
}
return true;
}
else if (message.Action == "DeleteArchive" && message.ArchiveId != null)
{
using (ArchiveTransferManager manager = new ArchiveTransferManager(Context.Mapping.AccessKey, Context.Mapping.SecretKey, Context.Mapping.Endpoint))
{
Context.Vault.deleteArchive(manager, message.ArchiveId);
}
return true;
}
else if (message.Action == "ArchiveRetrievalStarted" && message.ArchiveId != null)
{
DateTime scheduledAt = message.ScheduledAt != null && message.ScheduledAt.HasValue ? message.ScheduledAt.Value : DateTime.Now;
Context.Vault.recordDownloadScheduled(message.ArchiveId, message.JobId, scheduledAt);
return true;
}
else if (message.Action == "StartSynchronizing")
{
Sync.SynchronizingWithServer = true;
return true;
}
else if (message.Action == "Stop")
{
Sync.stop();
return true;
}
else if (message.Action == "ListJobs")
{
showResults(message, listJobs());
return true;
}
else if (message.Action == "DescribeJob" && message.JobId != null)
{
showResults(message, describeJob(message.JobId));
return true;
}
}
if (message.Action == "StatusCheck")
{
showResults(message, showStatus());
return true;
}
else if (message.Action == "Inactivate")
{
active = false;
return true;
}
else if (message.Action == "Activate")
{
active = true;
return true;
}
return false;
}
public ArchiveUploadHighLevel(string vaultName, Amazon.RegionEndpoint endpoint)
{
VaultName = vaultName;
manager = new ArchiveTransferManager(endpoint);
}
public void upload(ArchiveTransferManager manager, string vaultName, string basePath)
{
ArchiveId = manager.Upload(vaultName, ArchiveDescription, basePath + "\\" + RelativePath).ArchiveId;
CreationDate = DateTime.Now;
IsDirty = true;
}
public bool Create()
{
try
{
var manager = new ArchiveTransferManager(RegionEndpoint);
manager.CreateVault(Name);
return true;
}
catch (AmazonGlacierException e) { Console.WriteLine(e.Message); }
catch (AmazonServiceException e) { Console.WriteLine(e.Message); }
catch (Exception e) { Console.WriteLine(e.Message); }
return false;
}
