Esempio n. 1
0
        AWSArchiveResult CompleteMPU(string uploadID, AmazonGlacier client, List <string> partChecksumList, FileInfo fio)
        {
            try
            {
                long fileLength = fio.Length;

                fileLength = new FileInfo(archiveToUpload).Length;

                FileStream inputFile     = File.Open(archiveToUpload, FileMode.Open, FileAccess.Read);
                byte[]     treeHash      = Form1.ComputeSHA256TreeHash(inputFile);
                String     localChecksum = BitConverter.ToString(treeHash).Replace("-", "").ToLower();

                CompleteMultipartUploadRequest completeMPUrequest = new CompleteMultipartUploadRequest()
                {
                    UploadId    = uploadID,
                    ArchiveSize = fileLength.ToString(),
                    Checksum    = localChecksum,
                    VaultName   = vaultName
                };

                CompleteMultipartUploadResponse completeMPUresponse = client.CompleteMultipartUpload(completeMPUrequest);

                AWSArchiveResult ar = new AWSArchiveResult();
                ar.ArchiveID = completeMPUresponse.CompleteMultipartUploadResult.ArchiveId;
                ar.Checksum  = localChecksum;

                return(ar);
            }
            catch (Exception e)
            {
                Form1.log.Error(e.ToString());
                return(new AWSArchiveResult());
            }
        }
Esempio n. 2
0
        static List <string> UploadParts(string uploadID, AmazonGlacier client)
        {
            var  partChecksumList = new List <string>();
            long currentPosition  = 0;
            var  buffer           = new byte[Convert.ToInt32(partSize)];

            long fileLength = new FileInfo(BackupFilePath).Length;

            WriteFileUploadProgress(currentPosition, fileLength);
            using (var fileToUpload = new FileStream(BackupFilePath, FileMode.Open, FileAccess.Read))
            {
                while (fileToUpload.Position < fileLength)
                {
                    var uploadPartStream = GlacierUtils.CreatePartStream(fileToUpload, partSize);
                    var checksum         = TreeHashGenerator.CalculateTreeHash(uploadPartStream);
                    partChecksumList.Add(checksum);
                    // Upload part.
                    var uploadMPUrequest = new UploadMultipartPartRequest()
                    {
                        VaultName = VaultName,
                        Body      = uploadPartStream,
                        Checksum  = checksum,
                        UploadId  = uploadID
                    };
                    uploadMPUrequest.SetRange(currentPosition, currentPosition + uploadPartStream.Length - 1);
                    client.UploadMultipartPart(uploadMPUrequest);
                    currentPosition = currentPosition + uploadPartStream.Length;
                    WriteFileUploadProgress(currentPosition, fileLength);
                }
            }
            return(partChecksumList);
        }
Esempio n. 3
0
        public static bool checkVault(string vaultName)
        {
            bool result = false;

            try
            {
                client = new AmazonGlacierClient(Amazon.RegionEndpoint.USEast1);

                DescribeVaultRequest describeVaultRequest = new DescribeVaultRequest()
                {
                    VaultName = vaultName
                };
                DescribeVaultResponse describeVaultResponse = client.DescribeVault(describeVaultRequest);
                DescribeVaultResult   describeVaultResult   = describeVaultResponse.DescribeVaultResult;
                Console.WriteLine("\nVault description...");
                Console.WriteLine(
                    "\nVaultName: " + describeVaultResult.VaultName +
                    "\nVaultARN: " + describeVaultResult.VaultARN +
                    "\nVaultCreationDate: " + describeVaultResult.CreationDate +
                    "\nNumberOfArchives: " + describeVaultResult.NumberOfArchives +
                    "\nSizeInBytes: " + describeVaultResult.SizeInBytes +
                    "\nLastInventoryDate: " + describeVaultResult.LastInventoryDate
                    );
                result = true;
            }
            catch (AmazonGlacierException e)
            { Console.WriteLine(e.Message); result = false; }
            catch (AmazonServiceException e)
            { Console.WriteLine(e.Message); result = false; }
            catch (Exception e)
            { Console.WriteLine(e.Message); result = false; }

            return(result);
        }
Esempio n. 4
0
        string InitiateMultipartUpload(AmazonGlacier client)
        {
            InitiateMultipartUploadRequest initiateMPUrequest = new InitiateMultipartUploadRequest()
            {
                VaultName          = vaultName,
                PartSize           = partSize,
                ArchiveDescription = archiveDescription
            };

            InitiateMultipartUploadResponse initiateMPUresponse = client.InitiateMultipartUpload(initiateMPUrequest);

            return(initiateMPUresponse.InitiateMultipartUploadResult.UploadId);
        }
Esempio n. 5
0
        private bool retryUpload(object f_object)
        {
            try
            {
                ThreadData    objData          = null;
                AmazonGlacier client           = null;
                Stream        uploadPartStream = null;

                objData = (ThreadData)f_object;

                string uploadID = objData.uploadID;
                client = objData.client;
                long currentPosition = objData.currentPosition;
                Form1.log.Info("Trying to upload Part :" + Convert.ToString(objData.currentPosition));

                //For the last one we need to make sure the buffer is the right size?
                //The uploadMPUrequest.SetRange probably takes care of this.

                int memoryBufferIndex = 0;                                      //The index into buffer at which the stream begin
                int memoryBuffercount = (int)(objData.uploadPartStream.Length); //The length of the stream in bytes.

                uploadPartStream = new MemoryStream(objData.buffer, memoryBufferIndex, memoryBuffercount);

                //To ensure that part data is not corrupted in transmission, you compute a SHA256 tree
                // hash of the part and include it in your request. Upon receiving the part data, Amazon Glacier also computes a SHA256 tree hash.
                //If these hash values don't match, the operation fails. For information about computing a SHA256 tree hash, see Computing Checksums
                string checksum = TreeHashGenerator.CalculateTreeHash(uploadPartStream);

                SHA256ConcurrentQueue.Enqueue(checksum);

                UploadMultipartPartRequest uploadMPUrequest = new UploadMultipartPartRequest()
                {
                    VaultName = vaultName,
                    Body      = uploadPartStream,
                    Checksum  = checksum,
                    UploadId  = uploadID
                };

                uploadMPUrequest.SetRange(currentPosition, currentPosition + objData.uploadPartStream.Length - 1);

                UploadMultipartPartResponse mpr = client.UploadMultipartPart(uploadMPUrequest);
                Form1.log.Info("Retry Success " + Convert.ToString(mpr.ContentLength) + "bytes" + " for Part  :" + Convert.ToString(objData.currentPosition));
                return(true);
            }
            catch (Exception ex)
            {
                Form1.log.Error(ex.ToString());
                return(false);
            }
        }
Esempio n. 6
0
        static string CompleteMPU(string uploadID, AmazonGlacier client, List <string> partChecksumList)
        {
            long fileLength         = new FileInfo(BackupFilePath).Length;
            var  completeMPUrequest = new CompleteMultipartUploadRequest()
            {
                UploadId    = uploadID,
                ArchiveSize = fileLength.ToString(),
                Checksum    = TreeHashGenerator.CalculateTreeHash(partChecksumList),
                VaultName   = VaultName
            };

            var completeMPUresponse = client.CompleteMultipartUpload(completeMPUrequest);

            return(completeMPUresponse.CompleteMultipartUploadResult.ArchiveId);
        }
Esempio n. 7
0
        private static void DownloadOutput(string jobId, AmazonGlacier client, AWSArchiveRquest request)
        {
            GetJobOutputRequest getJobOutputRequest = new GetJobOutputRequest()
            {
                JobId     = jobId,
                VaultName = vaultName
            };
            GetJobOutputResponse getJobOutputResponse = client.GetJobOutput(getJobOutputRequest);
            GetJobOutputResult   result = getJobOutputResponse.GetJobOutputResult;

            using (Stream webStream = result.Body)
            {
                using (Stream fileToSave = File.OpenWrite(request.Description))
                {
                    CopyStream(webStream, fileToSave);
                }
            }
        }
Esempio n. 8
0
        static void RetrieveArchive(AmazonGlacier client, AWSArchiveRquest request)
        {
            // Initiate job.
            InitiateJobRequest initJobRequest = new InitiateJobRequest()
            {
                VaultName     = vaultName,
                JobParameters = new JobParameters()
                {
                    Type        = "archive-retrieval",
                    ArchiveId   = request.ArchiveID,
                    Description = "This job is to download archive updated as part of getting started",
                    SNSTopic    = topicArn,
                }
            };
            InitiateJobResponse initJobResponse = client.InitiateJob(initJobRequest);
            string jobId = initJobResponse.InitiateJobResult.JobId;

            // Check queue for a message and if job completed successfully, download archive.
            ProcessQueue(jobId, client, request);
        }
Esempio n. 9
0
        private static void ProcessQueue(string jobId, AmazonGlacier client, AWSArchiveRquest request)
        {
            var receiveMessageRequest = new ReceiveMessageRequest()
            {
                QueueUrl = queueUrl, MaxNumberOfMessages = 1
            };
            bool jobDone = false;

            while (!jobDone)
            {
                var receiveMessageResponse = sqsClient.ReceiveMessage(receiveMessageRequest);
                if (receiveMessageResponse.ReceiveMessageResult.Message.Count == 0)
                {
                    Thread.Sleep(1000 * 60);
                    continue;
                }
                Amazon.SQS.Model.Message    message    = receiveMessageResponse.ReceiveMessageResult.Message[0];
                Dictionary <string, string> outerLayer = JsonConvert.DeserializeObject <Dictionary <string, string> >(message.Body);
                Dictionary <string, string> fields     = JsonConvert.DeserializeObject <Dictionary <string, string> >(outerLayer["Message"]);
                string statusCode = fields["StatusCode"] as string;
                if (string.Equals(statusCode, GlacierUtils.JOB_STATUS_SUCCEEDED, StringComparison.InvariantCultureIgnoreCase))
                {
                    Console.WriteLine("Downloading job output");
                    DownloadOutput(jobId, client, request); // This where we save job output to the specified file location.
                }
                else if (string.Equals(statusCode, GlacierUtils.JOB_STATUS_FAILED, StringComparison.InvariantCultureIgnoreCase))
                {
                    Console.WriteLine("Job failed... cannot download the archive.");
                }
                jobDone = true;
                sqsClient.DeleteMessage(new DeleteMessageRequest()
                {
                    QueueUrl = queueUrl, ReceiptHandle = message.ReceiptHandle
                });
            }
        }
Esempio n. 10
0
        private void ThreadUpload(object f_object)
        {
            ThreadData    objData          = null;
            AmazonGlacier client           = null;
            Stream        uploadPartStream = null;

            try
            {
                objData = (ThreadData)f_object;

                string uploadID = objData.uploadID;
                client = objData.client;
                long currentPosition = objData.currentPosition;
                Form1.log.Info("Trying to upload Part :" + Convert.ToString(objData.currentPosition));

                //For the last one we need to make sure the buffer is the right size?
                //The uploadMPUrequest.SetRange probably takes care of this.

                int memoryBufferIndex = 0;                                      //The index into buffer at which the stream begin
                int memoryBuffercount = (int)(objData.uploadPartStream.Length); //The length of the stream in bytes.

                uploadPartStream = new MemoryStream(objData.buffer, memoryBufferIndex, memoryBuffercount);

                //To ensure that part data is not corrupted in transmission, you compute a SHA256 tree
                // hash of the part and include it in your request. Upon receiving the part data, Amazon Glacier also computes a SHA256 tree hash.
                //If these hash values don't match, the operation fails. For information about computing a SHA256 tree hash, see Computing Checksums
                string checksum = TreeHashGenerator.CalculateTreeHash(uploadPartStream);

                SHA256ConcurrentQueue.Enqueue(checksum);

                UploadMultipartPartRequest uploadMPUrequest = new UploadMultipartPartRequest()
                {
                    VaultName = vaultName,
                    Body      = uploadPartStream,
                    Checksum  = checksum,
                    UploadId  = uploadID
                };

                uploadMPUrequest.SetRange(currentPosition, currentPosition + objData.uploadPartStream.Length - 1);

                UploadMultipartPartResponse mpr = client.UploadMultipartPart(uploadMPUrequest);

                Form1.log.Info("Sent " + Convert.ToString(mpr.ContentLength) + "bytes" + " for Part  :" + Convert.ToString(objData.currentPosition));
            }
            catch (Exception e)
            {
                Form1.log.Error(e.ToString());
                Form1.log.Error(e.StackTrace);
                Form1.log.Info("Retrying Part " + Convert.ToString(objData.currentPosition));

                //Retrying up to 10 times - waiting longer each try
                {
                    int  fv = 0;
                    bool successfulPartUpload = false;
                    while (fv < 10 && successfulPartUpload == false)
                    {
                        successfulPartUpload = retryUpload(f_object);
                        Thread.Sleep(4000 * fv);
                    }
                }
            }
            finally
            {
                if (Interlocked.Decrement(ref ActiveWorkerCount) <= 0)
                {
                    AllWorkerCompletedEvent.Set();
                }

                uploadPartStream = null;
                f_object         = null;
                objData          = null;
                client           = null;
            }
        }
Esempio n. 11
0
        List <string> UploadParts(string uploadID, AmazonGlacier client)
        {
            List <string> partChecksumList = new List <string>();
            long          currentPosition  = 0;

            var buffer = new byte[Convert.ToInt32(partSize)];

            long fileLength = new FileInfo(archiveToUpload).Length;

            ThreadPool.SetMaxThreads(25, 25);

            List <ThreadData> arThreadObj = new List <ThreadData>();

            FileStream fileToUpload = new FileStream(archiveToUpload, FileMode.Open, FileAccess.Read);

            //Beware - we create memory buffers for the entire file at once.
            //BBCREVISIT - use a queue for the threads and pick off that as the ThreadPool frees up resources
            while (currentPosition < fileLength)
            {
                Stream uploadPartStream = GlacierUtils.CreatePartStream(fileToUpload, partSize);

                ThreadData objData = new ThreadData();
                objData.uploadID         = uploadID;
                objData.client           = client;
                objData.currentPosition  = currentPosition;
                objData.uploadPartStream = uploadPartStream;
                objData.buffer           = new byte[Convert.ToInt32(partSize)];
                int read = 0;
                try
                {
                    read = fileToUpload.Read(objData.buffer, (int)0, (int)partSize);
                }
                catch (Exception e)
                {
                    Form1.log.Error(e.ToString());
                }

                if (read == -1)
                {
                    Form1.log.Info("Nothing to read :  fileLength % partSize ==0");
                    break;
                }

                arThreadObj.Add(objData);

                Form1.log.Info("Created Part : " + Convert.ToString(currentPosition) + " of Length " + uploadPartStream.Length);

                if (read != uploadPartStream.Length)
                {
                    Console.WriteLine("We have a problem Houston");
                }
                currentPosition = currentPosition + uploadPartStream.Length;//We are not using the stream right now.
            }

            for (int ic = 0; ic < arThreadObj.Count; ic++)
            {
                Interlocked.Increment(ref ActiveWorkerCount);
                ThreadData objData = arThreadObj[ic];
                ThreadPool.QueueUserWorkItem(ThreadUpload, objData);
            }

            AllWorkerCompletedEvent.WaitOne();

            partChecksumList = SHA256ConcurrentQueue.ToList();

            fileToUpload.Close();

            return(partChecksumList);
        }
Esempio n. 12
0
        static List<string> UploadParts(string uploadID, AmazonGlacier client)
        {
            var partChecksumList = new List<string>();
            long currentPosition = 0;
            var buffer = new byte[Convert.ToInt32(partSize)];

            long fileLength = new FileInfo(BackupFilePath).Length;
            WriteFileUploadProgress(currentPosition, fileLength);
            using (var fileToUpload = new FileStream(BackupFilePath, FileMode.Open, FileAccess.Read))
            {
                while (fileToUpload.Position < fileLength)
                {
                    var uploadPartStream = GlacierUtils.CreatePartStream(fileToUpload, partSize);
                    var checksum = TreeHashGenerator.CalculateTreeHash(uploadPartStream);
                    partChecksumList.Add(checksum);
                    // Upload part.
                    var uploadMPUrequest = new UploadMultipartPartRequest()
                    {
                        VaultName = VaultName,
                        Body = uploadPartStream,
                        Checksum = checksum,
                        UploadId = uploadID
                    };
                    uploadMPUrequest.SetRange(currentPosition, currentPosition + uploadPartStream.Length - 1);
                    client.UploadMultipartPart(uploadMPUrequest);
                    currentPosition = currentPosition + uploadPartStream.Length;
                    WriteFileUploadProgress(currentPosition, fileLength);
                }
            }
            return partChecksumList;
        }
 /// <summary>
 /// Constructs an ArchiveTransferManager object for the specified Amazon Glacier region endpoint using the credentials
 /// loaded from the application's default configuration, and if unsuccessful from the Instance Profile service on an EC2 instance.
 ///
 /// Example App.config with credentials set.
 /// <code>
 /// &lt;?xml version="1.0" encoding="utf-8" ?&gt;
 /// &lt;configuration&gt;
 ///     &lt;appSettings&gt;
 ///         &lt;add key="AWSAccessKey" value="********************"/&gt;
 ///         &lt;add key="AWSSecretKey" value="****************************************"/&gt;
 ///     &lt;/appSettings&gt;
 /// &lt;/configuration&gt;
 /// </code>
 ///
 /// </summary>
 /// <param name="region">Amazon Glacier region endpoint</param>
 public ArchiveTransferManager(RegionEndpoint region)
 {
     this.glacierClient = new AmazonGlacierClient(region);
     this.shouldDispose = true;
 }
Esempio n. 14
0
        static string CompleteMPU(string uploadID, AmazonGlacier client, List<string> partChecksumList)
        {
            long fileLength = new FileInfo(BackupFilePath).Length;
            var completeMPUrequest = new CompleteMultipartUploadRequest()
            {
                UploadId = uploadID,
                ArchiveSize = fileLength.ToString(),
                Checksum = TreeHashGenerator.CalculateTreeHash(partChecksumList),
                VaultName = VaultName
            };

            var completeMPUresponse = client.CompleteMultipartUpload(completeMPUrequest);
            return completeMPUresponse.CompleteMultipartUploadResult.ArchiveId;
        }
Esempio n. 15
0
 /// <summary>
 /// Constructs an ArchiveTransferManager object for the specified Amazon Glacier region endpoint using the credentials
 /// loaded from the application's default configuration, and if unsuccessful from the Instance Profile service on an EC2 instance.
 /// 
 /// Example App.config with credentials set. 
 /// <code>
 /// &lt;?xml version="1.0" encoding="utf-8" ?&gt;
 /// &lt;configuration&gt;
 ///     &lt;appSettings&gt;
 ///         &lt;add key="AWSAccessKey" value="********************"/&gt;
 ///         &lt;add key="AWSSecretKey" value="****************************************"/&gt;
 ///     &lt;/appSettings&gt;
 /// &lt;/configuration&gt;
 /// </code>
 ///
 /// </summary>
 /// <param name="region">Amazon Glacier region endpoint</param>
 public ArchiveTransferManager(RegionEndpoint region)
 {
     this.glacierClient = new AmazonGlacierClient(region);
     this.shouldDispose = true;
 }
 /// <summary>
 /// Constructs an ArchiveTransferManager object using an existing Amazon Glacier client.
 /// </summary>
 /// <param name="glacier">An AmazonGlacier client that used to make service calls.</param>
 public ArchiveTransferManager(AmazonGlacier glacier)
 {
     this.glacierClient = glacier;
     this.shouldDispose = false;
 }
Esempio n. 17
0
 /// <summary>
 /// Constructs an ArchiveTransferManager object using an existing Amazon Glacier client.
 /// </summary>
 /// <param name="glacier">An AmazonGlacier client that used to make service calls.</param>
 public ArchiveTransferManager(AmazonGlacier glacier)
 {
     this.glacierClient = glacier;
     this.shouldDispose = false;
 }
Esempio n. 18
0
        static string InitiateMultipartUpload(AmazonGlacier client)
        {
            var initiateMPUrequest = new InitiateMultipartUploadRequest()
            {
                VaultName = VaultName,
                PartSize = partSize,
                ArchiveDescription = ArchiveDescription
            };

            var initiateMPUresponse = client.InitiateMultipartUpload(initiateMPUrequest);

            return initiateMPUresponse.InitiateMultipartUploadResult.UploadId;
        }