public static GlacierResult ProcessQueue(Topic topic)
{
// Check for notifications on topic and process any message
try
{
var settings = SettingsManager.GetSettings();
using (var client = new AmazonGlacierClient(
settings.AWSAccessKeyID,
settings.AWSSecretAccessKey,
RegionEndpoint.GetBySystemName(settings.AWSS3Region.SystemName)))
{
var receiveMessageRequest = new ReceiveMessageRequest {
QueueUrl = topic.QueueUrl, MaxNumberOfMessages = 1
};
var sqsClient = new AmazonSQSClient(settings.AWSAccessKeyID, settings.AWSSecretAccessKey, RegionEndpoint.GetBySystemName(settings.AWSS3Region.SystemName));
var receiveMessageResponse = sqsClient.ReceiveMessage(receiveMessageRequest);
if (receiveMessageResponse.Messages.Count == 0)
{
topic.Status = GlacierResult.Incomplete;
SaveTopicFile(topic);
return(topic.Status);
}
// Process message
string status = GetResponseStatus(receiveMessageResponse);
if (string.Equals(status, GlacierUtils.JOB_STATUS_SUCCEEDED, StringComparison.InvariantCultureIgnoreCase))
{
DownloadGlacierJobOutput(topic.JobId, client, settings.AWSGlacierVault, topic.GetOutputFile());
Debug.WriteLine($"Downloaded job output to {topic.GetOutputFile()}");
if (topic.ArchiveId != null)
{
DownloadSuccess?.Invoke($"Glacier archive was downloaded to {topic.GetOutputFile()}");
}
DeleteTopic(topic);
return(GlacierResult.Completed);
}
else if (string.Equals(status, GlacierUtils.JOB_STATUS_FAILED, StringComparison.InvariantCultureIgnoreCase))
{
DownloadError?.Invoke("Job failed, cannot download the file");
DeleteTopic(topic);
return(GlacierResult.JobFailed);
}
else if (string.Equals(status, GlacierUtils.JOB_STATUS_INPROGRESS, StringComparison.InvariantCultureIgnoreCase))
{
DownloadWarning?.Invoke("Job in progress, Queue ARN: " + topic.QueueARN);
DeleteTopic(topic);
return(GlacierResult.JobInProgress);
}
else
{
DeleteTopic(topic);
return(GlacierResult.Error);
}
}
}
catch (AmazonServiceException azex)
{
// Handle specific potential errors here
Debug.WriteLine("AmazonServiceException " + azex.Message);
if (azex.StatusCode == System.Net.HttpStatusCode.Forbidden)
{
// Invalid credentials
BackupError?.Invoke("Invalid AWS credentials were provided while connecting");
return(GlacierResult.Incomplete);
}
if (azex.InnerException != null &&
azex.InnerException is System.Net.WebException &&
((System.Net.WebException)azex.InnerException).Status == System.Net.WebExceptionStatus.NameResolutionFailure)
{
// Not connected to internet
BackupError?.Invoke("Network connection failure");
return(GlacierResult.Incomplete);
}
if (azex.InnerException != null &&
azex.InnerException is System.Net.WebException)
{
// Network errors
BackupError?.Invoke($"A network error occurred ({((System.Net.WebException)azex.InnerException).Status})");
return(GlacierResult.Incomplete);
}
if (azex.StatusCode == System.Net.HttpStatusCode.BadRequest
//&& topic.Status == GlacierResult.JobRequested
&& azex.Message.Contains("The specified queue does not exist") &&
DateTime.Now - topic.DateRequested < new TimeSpan(24, 0, 0))
{
// Job was recently requested and the queue has not been created yet
Debug.WriteLine("Job request may be in progress");
return(GlacierResult.JobRequested);
}
// TODO Check expiry?
// Glacier ref: "A job ID will not expire for at least 24 hours after Amazon Glacier completes the job."
DeleteTopic(topic);
BackupWarning?.Invoke("An AWS Glacier job has expired, a new job will be issued");
// Reissue expired job
InitiateGlacierJob(topic);
return(topic.Status);
}
catch (Exception ex)
{
DeleteTopic(topic);
throw ex;
}
}
/// <summary>
/// Do the backing up of the discovered files
/// </summary>
/// <param name="archiveDir">A directory to backup to</param>
static public void CopyFiles(string archiveDir = null)
{
var settings = SettingsManager.GetSettings();
var tempDir = GetTempDirectory();
var filesName = Path.Combine(tempDir, DiscoveredFileName);
if (!File.Exists(filesName))
{
BackupWarning?.Invoke("No new or modified files were discovered");
return; // Nothing to do
}
var files = File.ReadAllLines(filesName);
if (files.Length == 0)
{
BackupWarning?.Invoke("No new or modified files were discovered");
return; // Nothing to do
}
if (archiveDir == null)
{
archiveDir = settings.FileSystemDirectory;
}
// Has user set an archive folder?
if (String.IsNullOrEmpty(archiveDir))
{
archiveDir = Path.Combine(tempDir, ArchiveFolder);
}
// Make the archive
var fileSource = DoCopy(archiveDir, files);
// Upload the zip archive
if (settings.IsS3BucketEnabled)
{
UploadS3Archive(settings, fileSource.ZipFileArchive);
BackupSuccess?.Invoke($"Backup uploaded to S3 Bucket {settings.AWSS3Bucket}");
// Clean up if the user didn't want the File System option
if (!settings.IsFileSystemEnabled)
{
File.Delete(fileSource.ZipFileArchive);
}
}
if (settings.IsGlacierEnabled)
{
UploadGlacierArchive(settings, fileSource.ZipFileArchive);
BackupSuccess?.Invoke($"Backup uploaded to Glacier Vault {settings.AWSGlacierVault}");
// Clean up if the user didn't want the File System option
if (!settings.IsFileSystemEnabled)
{
File.Delete(fileSource.ZipFileArchive);
}
}
try
{
// Clean up source directory created for zip archive
DeleteZipSource(fileSource.UniqueArchiveDir);
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
BackupWarning?.Invoke($"An error occurred removing the temporary archive files: {fileSource.UniqueArchiveDir}");
}
}
/// <summary>
/// Copy files to the given directory and return the zip file name
/// </summary>
private static FileSource DoCopy(string archiveDir, string[] files)
{
var fileSource = new FileSource {
UniqueArchiveDir = GetArchiveUniqueName(archiveDir)
};
foreach (var line in files)
{
try
{
// Copy file and append to processing file
var filepath = (from f in line.Split(',') select f.Trim()).First();
var subpath = (from f in line.Split(',') select f.Trim()).Last();
// Create the subpath in archive
var splits = subpath.Split('\\');
var subpathDir = "";
for (int i = 0; i < splits.Length - 1; i++)
{
subpathDir += "\\" + splits[i];
}
var newDir = Path.Combine(fileSource.UniqueArchiveDir, subpathDir.TrimStart('\\'));
var di = Directory.CreateDirectory(newDir);
//Debug.Write(di);
try
{
var dest = Path.Combine(fileSource.UniqueArchiveDir, subpath.TrimStart('\\'));
File.Copy(filepath, dest, true);
File.SetAttributes(dest, FileAttributes.Normal);
}
catch (UnauthorizedAccessException ex)
{
Debug.WriteLine("UnauthorizedAccessException: " + ex.Message);
BackupWarning?.Invoke($"Unauthorized access error for {filepath}. File will be skipped.");
}
catch (IOException io)
{
Debug.WriteLine("IOException: " + io.Message);
BackupWarning?.Invoke($"IO Error for {filepath}. File will be skipped.");
}
fileSource.FileCount++;
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
throw ex;
}
}
if (fileSource.FileCount > 0)
{
try
{
ZipFile.CreateFromDirectory(fileSource.UniqueArchiveDir, fileSource.ZipFileArchive, CompressionLevel.Optimal, false);
// Clean up folders used to make the zip archive
//DeleteZipSource(fileSource.UniqueArchiveDir);
BackupSuccess?.Invoke($"{fileSource.FileCount} file {(fileSource.FileCount == 1 ? String.Empty : "s")} copied to {fileSource.ZipFileArchive}");
return(fileSource);
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
throw ex;
}
}
return(null);
}
