예제 #1
0
        // Parse the input request



        /// <summary>
        /// //////////////////////////////Create a new bucket//////////////////////////////
        /// </summary>
        /// <param name="client"></param>
        public static void CreateBucket(AmazonS3 client, string cmap)
        {
            Console.Out.WriteLine("Checking S3 bucket with name " + cmap);

            ListBucketsResponse response = client.ListBuckets();

            bool found = false;

            foreach (S3Bucket bucket in response.Buckets)
            {
                if (bucket.BucketName == cmap)
                {
                    Console.Out.WriteLine("   Bucket found will not create it.");
                    found = true;
                    break;
                }
            }

            if (found == false)
            {
                Console.Out.WriteLine("   Bucket not found will create it.");

                client.PutBucket(new PutBucketRequest().WithBucketName(cmap));

                Console.Out.WriteLine("Created S3 bucket with name " + cmap);
            }
        }
예제 #2
0
파일: Bucket.cs 프로젝트: sojanml/eX-Portal
        public bool CreateBucket()
        {
            config.ServiceURL = "s3.amazonaws.com";

            AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(
                am.PublicKey,
                am.PrivateKey
                );

            ListBucketsResponse response = client.ListBuckets();

            bool found = false;

            foreach (S3Bucket bucket in response.Buckets)
            {
                if (bucket.BucketName == "Drone")
                {
                    found = true;
                    break;
                }
            }

            if (found == false)
            {
                client.PutBucket(new PutBucketRequest().WithBucketName("Drone"));
            }
            return(found);
        }
예제 #3
0
파일: FtpUpload.cs 프로젝트: yarivat/Admin
        //public bool IsFolderExists(string accessKeyID, string secretAccessKey, string bucketName, string key, bool create)
        //{
        //    AmazonS3 client = GetS3Client(accessKeyID, secretAccessKey);
        //    ListObjectsRequest listRequest = new ListObjectsRequest();
        //    listRequest.WithBucketName(bucketName)
        //    .WithPrefix(key);

        //    // get all objects inside the "folder"
        //    ListObjectsResponse objects = client.ListObjects(listRequest);
        //    foreach (S3Object s3o in objects.S3Objects)
        //    {
        //        // get the acl of the object
        //        GetACLRequest aclRequest = new GetACLRequest();
        //        aclRequest.WithBucketName("thebucket")
        //        .WithKey(s3o.Key);
        //        GetACLResponse getAclResponse = client.GetACL(aclRequest);

        //        //// copy the object without acl
        //        //string newKey = s3o.Key.Replace(oldOWnerId.ToString(), newOwnerId.ToString());
        //        //CopyObjectRequest copyRequest = new CopyObjectRequest();
        //        //copyRequest.SourceBucket = "thebucket";
        //        //copyRequest.DestinationBucket = "thebucket";
        //        //copyRequest.WithSourceKey(s3o.Key)
        //        //.WithDestinationKey(newKey);
        //        //S3Response copyResponse = client.CopyObject(copyRequest);
        //        //// set the acl of the newly made object
        //        //SetACLRequest setAclRequest = new SetACLRequest();
        //        //setAclRequest.WithBucketName("ytimusic")
        //        //.WithKey(newKey)
        //        //.WithACL(getAclResponse.AccessControlList);
        //        //SetACLResponse setAclRespone = client.SetACL(setAclRequest);
        //        //DeleteObjectRequest deleteRequest = new DeleteObjectRequest();
        //        //deleteRequest.WithBucketName("thebucket")
        //        //.WithKey(s3o.Key);
        //        //DeleteObjectResponse deleteResponse = client.DeleteObject(deleteRequest);

        //    }

        //    return false;
        //}

        /// <summary>
        /// Check Azure connection
        /// </summary>
        /// <param name="azureAccountName"></param>
        /// <param name="azureAccountKey"></param>
        /// <returns></returns>
        public bool IsBucketNameExists(string accessKeyID, string secretAccessKey, string bucketName, bool create)
        {
            try
            {
                AmazonS3            client   = GetS3Client(accessKeyID, secretAccessKey);
                ListBucketsResponse response = client.ListBuckets();
                bool found = false;
                foreach (S3Bucket bucket in response.Buckets)
                {
                    if (bucket.BucketName == bucketName)
                    {
                        found = true;
                        break;
                    }
                }
                if (found == false)
                {
                    if (create)
                    {
                        client.PutBucket(new PutBucketRequest().WithBucketName(bucketName));
                    }
                    else
                    {
                        throw new Exception("The bucket " + bucketName + " does not exists.");
                    }
                }
                return(true);
            }
            catch (Exception ex)
            {
                throw new DuradosException("AWS Connection failed, Error: " + ex.Message);
                //return false;
            }
        }
예제 #4
0
 static void ListingBuckets()
 {
     try
     {
         using (ListBucketsResponse response = client.ListBuckets())
         {
             foreach (S3Bucket bucket in response.Buckets)
             {
                 Console.WriteLine("You own Bucket with name: {0}", bucket.BucketName);
             }
         }
     }
     catch (AmazonS3Exception amazonS3Exception)
     {
         if (amazonS3Exception.ErrorCode != null &&
             (amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") ||
              amazonS3Exception.ErrorCode.Equals("InvalidSecurity")))
         {
             Console.WriteLine("Please check the provided AWS Credentials.");
             Console.WriteLine("If you haven't signed up for Amazon S3, please visit http://aws.amazon.com/s3");
         }
         else
         {
             Console.WriteLine("An Error, number {0}, occurred when listing buckets with the message '{1}", amazonS3Exception.ErrorCode, amazonS3Exception.Message);
         }
     }
 }
예제 #5
0
        private S3Bucket GetBucket(AmazonS3 client)
        {
            ListBucketsRequest  listRequest = new ListBucketsRequest();
            ListBucketsResponse response    = client.ListBuckets(listRequest);

            return(response.Buckets
                   .Where(candidate => candidate.BucketName == this.BucketName)
                   .FirstOrDefault());
        }
예제 #6
0
        private void WriteS3Info()
        {
            StringBuilder       output   = new StringBuilder();
            ListBucketsResponse response = s3.ListBuckets();

            if (response.Buckets != null && response.Buckets.Count > 0)
            {
                foreach (S3Bucket theBucket in response.Buckets)
                {
                    output.AppendFormat("<li>{0}</li>", theBucket.BucketName);
                }
            }
            this.s3Placeholder.Text = output.ToString();
        }
예제 #7
0
        public static void ClassInitialize(TestContext context)
        {
            try
            {
                client = ClientTests.CreateClient();

                var buckets = client.ListBuckets();
                bucket = buckets.Buckets.First();
            }
            catch (Exception e)
            {
                Assert.Inconclusive("prerequisite: unable to create client or bucket.  Error: {0}", e.Message);
            }
        }
        public static void ClassInitialize(TestContext context)
        {
            try
            {
                client = ClientTests.CreateClient();

                var buckets = client.ListBuckets();
                bucket = buckets.Buckets.First();
            }
            catch (Exception e)
            {
                Assert.Inconclusive("prerequisite: unable to create client or bucket.  Error: {0}", e.Message);
            }
        }
예제 #9
0
        public bool SaveFile(string sFolder, string sObjectKey, byte[] fileContent, bool bMakePublic)
        {
            try
            {
                AmazonS3 client = AWSClientFactory.CreateAmazonS3Client(S3ACCESSKEY, S3SECRETKEY);
                Amazon.S3.Transfer.TransferUtility uploader = new Amazon.S3.Transfer.TransferUtility(S3ACCESSKEY, S3SECRETKEY);
                string BUCKET_NAME = ConfigurationManager.AppSettings["AWSBUCKET"];

                ListBucketsResponse response = client.ListBuckets();
                bool found = false;
                foreach (S3Bucket bucket in response.Buckets)
                {
                    if (bucket.BucketName == BUCKET_NAME)
                    {
                        found = true;
                        break;
                    }
                }
                if (found == false)
                {
                    client.PutBucket(new PutBucketRequest().WithBucketName(BUCKET_NAME));
                }

                if (sFolder != "")
                {
                    sObjectKey = sFolder + "/" + sObjectKey;
                }

                System.IO.Stream stream = new System.IO.MemoryStream(fileContent);
                Amazon.S3.Transfer.TransferUtilityUploadRequest request = new Amazon.S3.Transfer.TransferUtilityUploadRequest();
                request.WithBucketName(BUCKET_NAME);
                request.WithKey(sObjectKey);
                request.WithInputStream(stream);
                request.WithTimeout(-1);
                if (bMakePublic)
                {
                    request.CannedACL = S3CannedACL.PublicRead;
                }
                uploader.Upload(request);
                return(true);
            }
            catch (Exception)
            {
                return(false);
            }
        }
예제 #10
0
 public static void CheckForBucket(string itemKey, AmazonS3 s3Client)
 {
     if (HttpContext.Current.User.Identity.IsAuthenticated)
     {
         string userBucketName = String.Format(Settings.Default.BucketNameFormat, HttpContext.Current.User.Identity.Name, itemKey);
         using (ListBucketsResponse listBucketsResponse = s3Client.ListBuckets())
         {
             S3Bucket bucket = listBucketsResponse.Buckets.FirstOrDefault(b => b.BucketName == userBucketName);
             if (bucket == null)
             {
                 PutBucketRequest putBucketRequest = new PutBucketRequest()
                     .WithBucketName(userBucketName);
                 PutBucketResponse putBucketResponse = s3Client.PutBucket(putBucketRequest);
                 putBucketResponse.Dispose();
             }
         }
     }
 }
예제 #11
0
        public static bool Initialize_S3_stuff()
        {
            Console.WriteLine("starting Initialize_S3_stuff()");

            s3_client = null;
            bucketName = null;
            try
            {
                if (!Utils.CFG.ContainsKey("s3_bucketName"))
                {
                    Console.WriteLine("param s3_bucketName is not found in ez3d.config");
                    return false;
                }
                s3_client = AWSClientFactory.CreateAmazonS3Client();

            //                ListBucketsRequest listBucketsRequest = new ListBucketsRequest();
                ListBucketsResponse response = s3_client.ListBuckets();

                foreach (S3Bucket bucket in response.Buckets)
                {
                    if (bucket.BucketName == (String)Utils.CFG["s3_bucketName"])
                    {
                        bucketName = bucket.BucketName;
                        Console.WriteLine("bucketName =" + bucketName);
                    }
                }

                if (bucketName == null)
                {
                    Console.WriteLine("(bucketName == null)");
                    return false;
                }

                Console.WriteLine("Initialize_S3_stuff fininshed succefully");
                return true;
            }
            catch (AmazonS3Exception e)
            {
                Console.WriteLine("AmazonS3Exception caught !!!");
                Console.WriteLine(e.Message);
                return false;
            }
        }
예제 #12
0
        /// <summary>
        /// Enumerate all the buckets.
        /// Note: AWS provides the list of buckets in a paged manner.
        /// A call to this function iterates over all the pages and sums up
        /// all the items.
        /// </summary>
        /// <returns>Array of bucket names</returns>
        public string[] EnumerateBuckets()
        {
            var response = _amazonS3.ListBuckets();

            return(response.Buckets.Select(b => b.BucketName).ToArray());
        }
예제 #13
0
        private static void CreateBucket(AmazonS3 client, string bucketname)
        {
            Console.Out.WriteLine("Checking S3 bucket with name " + bucketname);

            ListBucketsResponse response = client.ListBuckets();

            bool found = false;
            foreach (S3Bucket bucket in response.Buckets)
            {
                if (bucket.BucketName == bucketname)
                {
                    Console.Out.WriteLine("   Bucket found will not create it.");
                    found = true;
                    break;
                }
            }

            if (found == false)
            {
                Console.Out.WriteLine("   Bucket not found will create it.");

                client.PutBucket(new PutBucketRequest().WithBucketName(bucketname));

                Console.Out.WriteLine("Created S3 bucket with name " + bucketname);
            }
        }
예제 #14
0
        public List <string> GetAllBuckets()
        {
            ListBucketsResponse listBucketsResponse = _client.ListBuckets();

            return(listBucketsResponse.Buckets.Select(b => b.BucketName).ToList());
        }
예제 #15
0
파일: Program.cs 프로젝트: melnx/Bermuda
        public static string GetServiceOutput()
        {
            StringBuilder sb = new StringBuilder(1024);

            using (StringWriter sr = new StringWriter(sb))
            {
                sr.WriteLine("===========================================");
                sr.WriteLine("Welcome to the AWS .NET SDK!");
                sr.WriteLine("===========================================");

                // Print the number of Amazon EC2 instances.
                AmazonEC2 ec2 = AWSClientFactory.CreateAmazonEC2Client();
                DescribeInstancesRequest ec2Request = new DescribeInstancesRequest();

                try
                {
                    DescribeInstancesResponse ec2Response = ec2.DescribeInstances(ec2Request);
                    int numInstances = 0;
                    numInstances = ec2Response.DescribeInstancesResult.Reservation.Count;
                    sr.WriteLine("You have " + numInstances + " Amazon EC2 instance(s) running in the US-East (Northern Virginia) region.");
                }
                catch (AmazonEC2Exception ex)
                {
                    if (ex.ErrorCode != null && ex.ErrorCode.Equals("AuthFailure"))
                    {
                        sr.WriteLine("The account you are using is not signed up for Amazon EC2.");
                        sr.WriteLine("You can sign up for Amazon EC2 at http://aws.amazon.com/ec2");
                    }
                    else
                    {
                        sr.WriteLine("Caught Exception: " + ex.Message);
                        sr.WriteLine("Response Status Code: " + ex.StatusCode);
                        sr.WriteLine("Error Code: " + ex.ErrorCode);
                        sr.WriteLine("Error Type: " + ex.ErrorType);
                        sr.WriteLine("Request ID: " + ex.RequestId);
                        sr.WriteLine("XML: " + ex.XML);
                    }
                }
                sr.WriteLine();

                // Print the number of Amazon SimpleDB domains.
                AmazonSimpleDB     sdb        = AWSClientFactory.CreateAmazonSimpleDBClient();
                ListDomainsRequest sdbRequest = new ListDomainsRequest();

                try
                {
                    ListDomainsResponse sdbResponse = sdb.ListDomains(sdbRequest);

                    if (sdbResponse.IsSetListDomainsResult())
                    {
                        int numDomains = 0;
                        numDomains = sdbResponse.ListDomainsResult.DomainName.Count;
                        sr.WriteLine("You have " + numDomains + " Amazon SimpleDB domain(s) in the US-East (Northern Virginia) region.");
                    }
                }
                catch (AmazonSimpleDBException ex)
                {
                    if (ex.ErrorCode != null && ex.ErrorCode.Equals("AuthFailure"))
                    {
                        sr.WriteLine("The account you are using is not signed up for Amazon SimpleDB.");
                        sr.WriteLine("You can sign up for Amazon SimpleDB at http://aws.amazon.com/simpledb");
                    }
                    else
                    {
                        sr.WriteLine("Caught Exception: " + ex.Message);
                        sr.WriteLine("Response Status Code: " + ex.StatusCode);
                        sr.WriteLine("Error Code: " + ex.ErrorCode);
                        sr.WriteLine("Error Type: " + ex.ErrorType);
                        sr.WriteLine("Request ID: " + ex.RequestId);
                        sr.WriteLine("XML: " + ex.XML);
                    }
                }
                sr.WriteLine();

                // Print the number of Amazon S3 Buckets.
                AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client();

                try
                {
                    ListBucketsResponse response = s3Client.ListBuckets();
                    int numBuckets = 0;
                    if (response.Buckets != null &&
                        response.Buckets.Count > 0)
                    {
                        numBuckets = response.Buckets.Count;
                    }
                    sr.WriteLine("You have " + numBuckets + " Amazon S3 bucket(s) in the US Standard region.");
                }
                catch (AmazonS3Exception ex)
                {
                    if (ex.ErrorCode != null && (ex.ErrorCode.Equals("InvalidAccessKeyId") ||
                                                 ex.ErrorCode.Equals("InvalidSecurity")))
                    {
                        sr.WriteLine("Please check the provided AWS Credentials.");
                        sr.WriteLine("If you haven't signed up for Amazon S3, please visit http://aws.amazon.com/s3");
                    }
                    else
                    {
                        sr.WriteLine("Caught Exception: " + ex.Message);
                        sr.WriteLine("Response Status Code: " + ex.StatusCode);
                        sr.WriteLine("Error Code: " + ex.ErrorCode);
                        sr.WriteLine("Request ID: " + ex.RequestId);
                        sr.WriteLine("XML: " + ex.XML);
                    }
                }
                sr.WriteLine("Press any key to continue...");
            }
            return(sb.ToString());
        }
예제 #16
0
 private bool DoesBucketExist(AmazonS3 client)
 {
     using (ListBucketsResponse listBucketsResponse = client.ListBuckets())
         return(listBucketsResponse.Buckets.Any(b => b.BucketName == BucketName));
 }
예제 #17
0
 private bool DoesBucketExist(AmazonS3 client)
 {
     using (ListBucketsResponse listBucketsResponse = client.ListBuckets())
         return (listBucketsResponse.Buckets.Any(b => b.BucketName == BucketName));
 }
 private S3Bucket GetBucket(AmazonS3 client)
 {
     ListBucketsRequest listRequest = new ListBucketsRequest();
     ListBucketsResponse response = client.ListBuckets(listRequest);
     return response.Buckets
         .Where(candidate => candidate.BucketName == this.BucketName)
         .FirstOrDefault();
 }
예제 #19
0
        static void Main(string[] args)
        {
            startTime = DateTime.Now;

            //Catch exceptions
            AppDomain currentDomain = AppDomain.CurrentDomain;

            currentDomain.UnhandledException += new UnhandledExceptionEventHandler(MyHandler);

            //Catch ctrl+c to we can put out our summary
            Console.CancelKeyPress += (sender, eventArgs) =>
            {
                eventArgs.Cancel = false;
                WriteLog("!!CANCELLED!!");
                keepRunning = false;
                PrintSummary();
            };

            ServicePointManager.Expect100Continue = true;
            ServicePointManager.SecurityProtocol  = SecurityProtocolType.Ssl3;

            string strConfigFile = "CrossCloudBackup.xml";

            if (args.Length > 0)
            {
                strConfigFile = args[0];
            }

            if (!File.Exists(strConfigFile))
            {
                new XDocument(
                    new XDeclaration("1.0", "utf-8", "yes"),
                    new XComment("CrossCloudBackup Local Config File"),
                    new XElement("root",
                                 new XElement("AWSKey", "someValue"),
                                 new XElement("AWSSecret", "someValue"),
                                 new XElement("AWSRegion", "eu-west-1"),
                                 new XElement("RSUsername", "someValue"),
                                 new XElement("RSAPIKey", "someValue"),
                                 new XElement("RSUseServiceNet", "false"),
                                 new XElement("RSLocation", "UK"),
                                 new XElement("ExcludeBuckets", ""),
                                 new XElement("ExcludeContainers", ""),
                                 //new XElement("MirrorAll", "true"),  /TODO: Add Selective Sync
                                 new XElement("RSBackupContainer", "s3-backup"),
                                 new XElement("S3BackupBucket", "rs-backup")
                                 //new XElement("TransferThreads", "3") //TODO: Add Threading
                                 )
                    )
                .Save(strConfigFile);
                Console.WriteLine(strConfigFile + " not found, blank one created.");
                Console.WriteLine("Press enter to exit...");
                Console.ReadLine();
                Environment.Exit(1);
            }

            //We know the config file exists, so open and read values
            XDocument config = XDocument.Load(strConfigFile);

            //Get AWS config
            string         AWSKey    = config.Element("root").Element("AWSKey").Value;
            string         AWSSecret = config.Element("root").Element("AWSSecret").Value;
            RegionEndpoint region    = RegionEndpoint.EUWest1;

            switch (config.Element("root").Element("AWSRegion").Value)
            {
            case "eu-west-1":
                region = RegionEndpoint.EUWest1;
                break;

            case "sa-east-1":
                region = RegionEndpoint.SAEast1;
                break;

            case "us-east-1":
                region = RegionEndpoint.USEast1;
                break;

            case "ap-northeast-1":
                region = RegionEndpoint.APNortheast1;
                break;

            case "us-west-2":
                region = RegionEndpoint.USWest2;
                break;

            case "us-west-1":
                region = RegionEndpoint.USWest1;
                break;

            case "ap-southeast-1":
                region = RegionEndpoint.APSoutheast1;
                break;

            case "ap-southeast-2":
                region = RegionEndpoint.APSoutheast2;
                break;

            default:
                region = RegionEndpoint.EUWest1;
                break;
            }

            //Create a connection to S3
            WriteLog("Connecting to S3");
            S3Client = AWSClientFactory.CreateAmazonS3Client(AWSKey, AWSSecret, region);

            //Get RS config
            Rackspace.CloudFiles.Utils.AuthUrl rsRegion = Rackspace.CloudFiles.Utils.AuthUrl.US;
            switch (config.Element("root").Element("RSLocation").Value)
            {
            case "UK":
                rsRegion = Rackspace.CloudFiles.Utils.AuthUrl.UK;
                break;

            case "US":
                rsRegion = Rackspace.CloudFiles.Utils.AuthUrl.US;
                break;

            case "Mosso":
                rsRegion = Rackspace.CloudFiles.Utils.AuthUrl.Mosso;
                break;
            }

            //Create connection to Rackspace
            WriteLog("Connecting to Rackspace Cloud Files");
            RSConnection = new Connection(new UserCredentials(config.Element("root").Element("RSUsername").Value, config.Element("root").Element("RSAPIKey").Value, rsRegion), Convert.ToBoolean(config.Element("root").Element("RSUseServiceNet").Value));

            //Get exclusions
            string[] excludeBuckets    = config.Element("root").Element("ExcludeBuckets").Value.Split(',');
            string[] excludeContainers = config.Element("root").Element("ExcludeContainers").Value.Split(',');

            //First process all the S3 buckets and stream right into Rackspace container.
            WriteLog("Listing S3 Buckets");
            ListBucketsResponse response = S3Client.ListBuckets();

            WriteLog("Found " + response.Buckets.Count() + " buckets");
            foreach (S3Bucket bucket in response.Buckets)
            {
                if (bucket.BucketName == config.Element("root").Element("S3BackupBucket").Value)
                {
                    WriteLog("Skipping " + bucket.BucketName + " as backup folder");
                }
                else if (excludeBuckets.Contains(bucket.BucketName))
                {
                    WriteLog("Skipping " + bucket.BucketName + " as in exclusions");
                }
                else
                {
                    //We need to know if the bucket is in the right region, otherwise it will error
                    GetBucketLocationResponse locResponse = S3Client.GetBucketLocation(new GetBucketLocationRequest().WithBucketName(bucket.BucketName));
                    if (locResponse.Location == config.Element("root").Element("AWSRegion").Value)
                    {
                        WriteLog("Processing " + bucket.BucketName);
                        //Get list of files
                        ListObjectsRequest request = new ListObjectsRequest();
                        request.BucketName = bucket.BucketName;
                        do
                        {
                            ListObjectsResponse filesResponse = S3Client.ListObjects(request);
                            WriteLog("Found " + filesResponse.S3Objects.Count() + " files");
                            if (filesResponse.IsTruncated)
                            {
                                WriteLog("there are additional pages of files");
                            }
                            foreach (S3Object file in filesResponse.S3Objects)
                            {
                                bool bolTransfer = false;
                                //See if it exists on Rackspace
                                string uri = RSConnection.StorageUrl + "/" + config.Element("root").Element("RSBackupContainer").Value + "/" + bucket.BucketName + "/" + file.Key;
                                try
                                {
                                    var req = (HttpWebRequest)WebRequest.Create(uri);
                                    req.Headers.Add("X-Auth-Token", RSConnection.AuthToken);
                                    req.Method = "HEAD";

                                    //Compare Etags to see if we need to sync
                                    using (var resp = req.GetResponse() as HttpWebResponse)
                                    {
                                        if ("\"" + resp.Headers["eTag"] + "\"" != file.ETag)
                                        {
                                            bolTransfer = true;
                                        }
                                    }
                                }
                                catch (System.Net.WebException e)
                                {
                                    if (e.Status == WebExceptionStatus.ProtocolError && ((HttpWebResponse)e.Response).StatusCode == HttpStatusCode.NotFound)
                                    {
                                        //Item not found, so upload
                                        bolTransfer = true;
                                    }
                                    //WriteLog("End Request to " + uri);
                                }
                                if (file.StorageClass == "GLACIER")
                                {
                                    bolTransfer = false;                                 //We can't get things out of Glacier, but they aer still listed here.
                                }
                                if (bolTransfer)
                                {
                                    WriteLog("Syncing " + file.Key);
                                    using (GetObjectResponse getResponse = S3Client.GetObject(new GetObjectRequest().WithBucketName(bucket.BucketName).WithKey(file.Key)))
                                    {
                                        using (Stream s = getResponse.ResponseStream)
                                        {
                                            //We can stream right from s3 to CF, no need to store in memory or filesystem.
                                            var req = (HttpWebRequest)WebRequest.Create(uri);
                                            req.Headers.Add("X-Auth-Token", RSConnection.AuthToken);
                                            req.Method      = "PUT";
                                            req.SendChunked = true;
                                            req.AllowWriteStreamBuffering = false;
                                            req.Timeout = -1;

                                            using (Stream stream = req.GetRequestStream())
                                            {
                                                byte[] data      = new byte[8192];
                                                int    bytesRead = 0;
                                                while ((bytesRead = s.Read(data, 0, data.Length)) > 0)
                                                {
                                                    stream.Write(data, 0, bytesRead);
                                                }
                                                stream.Flush();
                                                stream.Close();
                                            }
                                            req.GetResponse().Close();
                                        }
                                    }
                                    intTransferred++;
                                    bytesTransferred += file.Size;
                                }
                                else
                                {
                                    WriteLog("Skipping " + file.Key);
                                    intSkipped++;
                                }

                                //Check our exit condition
                                if (!keepRunning)
                                {
                                    break;
                                }
                            }

                            //Loop if there is more than 1000 files
                            if (filesResponse.IsTruncated)
                            {
                                request.Marker = filesResponse.NextMarker;
                            }
                            else
                            {
                                request = null;
                            }

                            if (!keepRunning)
                            {
                                break;
                            }
                        } while (request != null);
                    }
                }
                if (!keepRunning)
                {
                    break;
                }
            }

            //Now get all the Rackspace containers and stream them to Amazon
            WriteLog("Listing CF Containers");
            List <string> lstContainers = RSConnection.GetContainers();

            WriteLog("Found " + lstContainers.Count() + " containers");
            foreach (string container in lstContainers)
            {
                if (container == config.Element("root").Element("RSBackupContainer").Value)
                {
                    WriteLog("Skipping " + container + " as backup folder");
                }
                else if (excludeContainers.Contains(container))
                {
                    WriteLog("Skipping " + container + " as in exclusions");
                }
                else
                {
                    WriteLog("Processing " + container);

                    XmlDocument containerInfo = RSConnection.GetContainerInformationXml(container);
                    do
                    {
                        int filesCount = containerInfo.GetElementsByTagName("object").Count;
                        WriteLog("Found " + filesCount + " files");
                        foreach (XmlNode file in containerInfo.GetElementsByTagName("object"))
                        {
                            bool   bolTransfer   = false;
                            string strBucketName = config.Element("root").Element("S3BackupBucket").Value;
                            string strKey        = container + file.SelectSingleNode("name").InnerText;
                            //See if the file exists on s3
                            try
                            {
                                GetObjectMetadataResponse metaResp = S3Client.GetObjectMetadata(new GetObjectMetadataRequest().WithBucketName(strBucketName).WithKey(strKey));
                                //Compare the etags
                                if (metaResp.ETag != "\"" + file.SelectSingleNode("hash").InnerText + "\"")
                                {
                                    bolTransfer = true;
                                }
                            }
                            catch (Amazon.S3.AmazonS3Exception e)
                            {
                                bolTransfer = true;
                            }

                            if (bolTransfer)
                            {
                                WriteLog("Syncing " + file.SelectSingleNode("name").InnerText);

                                //God the C# binding sucks, so let's stream manually
                                string uri = RSConnection.StorageUrl + "/" + container + "/" + file.SelectSingleNode("name").InnerText;
                                var    req = (HttpWebRequest)WebRequest.Create(uri);
                                req.Headers.Add("X-Auth-Token", RSConnection.AuthToken);
                                req.Method = "GET";

                                using (var resp = req.GetResponse() as HttpWebResponse)
                                {
                                    using (Stream s = resp.GetResponseStream())
                                    {
                                        string today = String.Format("{0:ddd,' 'dd' 'MMM' 'yyyy' 'HH':'mm':'ss' 'zz00}", DateTime.Now);

                                        string stringToSign = "PUT\n" +
                                                              "\n" +
                                                              file.SelectSingleNode("content_type").InnerText + "\n" +
                                                              "\n" +
                                                              "x-amz-date:" + today + "\n" +
                                                              "/" + strBucketName + "/" + strKey;

                                        Encoding ae               = new UTF8Encoding();
                                        HMACSHA1 signature        = new HMACSHA1(ae.GetBytes(AWSSecret));
                                        string   encodedCanonical = Convert.ToBase64String(signature.ComputeHash(ae.GetBytes(stringToSign)));

                                        string authHeader = "AWS " + AWSKey + ":" + encodedCanonical;

                                        string uriS3 = "https://" + strBucketName + ".s3.amazonaws.com/" + strKey;
                                        var    reqS3 = (HttpWebRequest)WebRequest.Create(uriS3);
                                        reqS3.Headers.Add("Authorization", authHeader);
                                        reqS3.Headers.Add("x-amz-date", today);
                                        reqS3.ContentType   = file.SelectSingleNode("content_type").InnerText;
                                        reqS3.ContentLength = Convert.ToInt32(file.SelectSingleNode("bytes").InnerText);
                                        reqS3.Method        = "PUT";

                                        reqS3.AllowWriteStreamBuffering = false;
                                        if (reqS3.ContentLength == -1L)
                                        {
                                            reqS3.SendChunked = true;
                                        }


                                        using (Stream streamS3 = reqS3.GetRequestStream())
                                        {
                                            byte[] data      = new byte[32768];
                                            int    bytesRead = 0;
                                            while ((bytesRead = s.Read(data, 0, data.Length)) > 0)
                                            {
                                                streamS3.Write(data, 0, bytesRead);
                                            }
                                            streamS3.Flush();
                                            streamS3.Close();
                                        }

                                        reqS3.GetResponse().Close();
                                    }
                                }


                                intTransferred++;
                                bytesTransferred += Convert.ToInt64(file.SelectSingleNode("bytes").InnerText);
                            }
                            else
                            {
                                WriteLog("Skipping " + file.SelectSingleNode("name").InnerText);
                                intSkipped++;
                            }

                            //Check our exit condition
                            if (!keepRunning)
                            {
                                break;
                            }
                        }

                        if (filesCount < 10000)
                        {
                            containerInfo = null;
                        }
                        else
                        {
                            //Fetch the next list, but the Rackspace binding doesn't support markers with XML responses....
                            try
                            {
                                string uri = RSConnection.StorageUrl + "/" + container + "?format=xml&marker=" + Uri.EscapeUriString(containerInfo.FirstChild.NextSibling.LastChild.SelectSingleNode("name").InnerText);
                                var    req = (HttpWebRequest)WebRequest.Create(uri);
                                req.Headers.Add("X-Auth-Token", RSConnection.AuthToken);
                                req.Method = "GET";

                                using (var resp = req.GetResponse() as HttpWebResponse)
                                {
                                    using (var reader = new System.IO.StreamReader(resp.GetResponseStream(), ASCIIEncoding.ASCII))
                                    {
                                        string responseText = reader.ReadToEnd();
                                        containerInfo.LoadXml(responseText);
                                    }
                                }
                            }
                            catch (System.Net.WebException e)
                            {
                                if (e.Status == WebExceptionStatus.ProtocolError && ((HttpWebResponse)e.Response).StatusCode == HttpStatusCode.NotFound)
                                {
                                    containerInfo = null;
                                }
                            }
                        }
                    } while (containerInfo != null);
                }
            }

            if (keepRunning)
            {
                WriteLog("Completed");
                PrintSummary();
            }
        }