private Amazon.RegionEndpoint GetBucketEndpoint(AWSCredentials awsCredentials, string awsBucketName)
{
Amazon.RegionEndpoint defaultResult = Amazon.RegionEndpoint.USEast1;
Amazon.RegionEndpoint result = null;
AmazonS3Client temporaryS3Client = null;
try
{
temporaryS3Client = new AmazonS3Client(awsCredentials, Amazon.RegionEndpoint.USEast1);
GetBucketLocationResponse bucketRegionResponse = temporaryS3Client.GetBucketLocation(awsBucketName);
result = Amazon.RegionEndpoint.GetBySystemName(bucketRegionResponse.Location.Value);
if (result == null || string.IsNullOrEmpty(result.SystemName))
{
result = defaultResult;
}
logger.Debug("The '{0}' bucket is located in the '{1}' region.", awsBucketName, result.DisplayName);
}
catch (Exception)
{
logger.Log(LogLevel.Warn, "Failed to discover the bucket location for '{0}'. Assuming '{1}'.", awsBucketName, defaultResult.DisplayName);
result = defaultResult;
}
finally
{
if (temporaryS3Client != null)
{
temporaryS3Client.Dispose();
}
}
return(result);
}
// Let us make sure that the bucket exists..
private async Task CheckCreateBucketAsync(string bucketName)
{
AmazonS3Client s3Client = GetS3Client();
try
{
GetBucketLocationResponse gblr = await s3Client.GetBucketLocationAsync(bucketName);
if (gblr != null)
{
Console.WriteLine("Bucket " + bucketName + " Exists @ " + gblr.Location.Value);
}
}
catch (AmazonS3Exception se)
{
// Got an Exception so bucket may not exists..
// Lets try creating the bucket..
Console.WriteLine("Bucket " + bucketName + " Does not Exist, error code-> " + se.ErrorCode);
PutBucketRequest pbr = new PutBucketRequest();
pbr.BucketName = bucketName;
PutBucketResponse pbrr = await s3Client.PutBucketAsync(pbr);
Console.WriteLine("Bucket " + bucketName + " Created in client region.");
}
}
void GetBucketLocationWebResponse(object sender, ResponseEventArgs args)
{
AmazonS3Exception response = args.Response as AmazonS3Exception;
S3Common.client.OnS3Response -= GetBucketLocationWebResponse;
if (null != response)
{
S3ErrorResponse(response);
return;
}
GetBucketLocationResponse locationResponse = args.Response as GetBucketLocationResponse;
if (null != locationResponse)
{
if (!string.IsNullOrEmpty(locationResponse.AmazonId2) &&
(!string.IsNullOrEmpty(locationResponse.RequestId)))
{
this.Dispatcher.BeginInvoke(() =>
{
GetBucketResult = String.Format(CultureInfo.InvariantCulture, "The Location of {0} Bucket: {1}", SelectedBucketName,
locationResponse.Location);
});
}
}
else
{
this.Dispatcher.BeginInvoke(() =>
{
GetBucketResult = string.Format(CultureInfo.InvariantCulture, "Error Processing GetBucketLocation Request");
});
}
}
public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext context)
{
GetBucketLocationResponse getBucketLocationResponse = new GetBucketLocationResponse();
UnmarshallResult(context, getBucketLocationResponse);
return(getBucketLocationResponse);
}
private static async Task <AmazonS3Client> SwitchClient(AmazonS3Client s3Client)
{
RegionEndpoint regionEp = null;
string uniqueKey =
string.Format(CultureInfo.InvariantCulture, "{0}_{1}", accessKeyId,
bucketName); // GetUniqueNameFromAccessKeyIdAndBucketName
GetBucketLocationResponse resp = await s3Client.GetBucketLocationAsync(bucketName);
if (resp == null)
{
return(s3Client);
}
else
{
regionEp = RegionToEndpoint(resp.Location);
}
if (regionEp != null && !regionEp.Equals(s3Client.Config.RegionEndpoint))
{
s3Client.Dispose();
AmazonS3Config config = new AmazonS3Config()
{
Timeout = TimeSpan.FromSeconds(300),
ReadWriteTimeout = TimeSpan.FromSeconds(600),
RegionEndpoint = regionEp
};
var credential = new BasicAWSCredentials(accessKeyId, secretAccessKey);
s3Client = new AmazonS3Client(credential, config);
}
return(s3Client);
}
private static void UnmarshallResult(XmlUnmarshallerContext context, GetBucketLocationResponse response)
{
int originalDepth = context.CurrentDepth;
int targetDepth = 1;
while (context.Read())
{
if (context.IsStartElement || context.IsAttribute)
{
if (context.TestExpression("LocationConstraint", targetDepth))
{
response.Location = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
}
else if (context.IsEndElement && context.CurrentDepth < originalDepth)
{
return;
}
}
return;
}
private async Task DiscoverAsync(AmazonS3Client client, S3Bucket bucket)
{
GetBucketLocationResponse bucketLocation = await client.GetBucketLocationAsync(bucket.BucketName);
string location = bucketLocation.Location.Value;
Children.Add(new AwsS3Bucket(bucket.BucketName, DisplayName, location));
}
private static void DisplayGetBucketLocationResponse(GetBucketLocationResponse getBucketLocationResponse)
{
var writer = Console.Out;
writer.WriteLine($"{nameof(getBucketLocationResponse.HttpStatusCode)}: {getBucketLocationResponse.HttpStatusCode}");
Construction.DisplayResponseMetadata(writer, getBucketLocationResponse.ResponseMetadata);
writer.WriteLine($"{nameof(getBucketLocationResponse.Location)}: {getBucketLocationResponse.Location}");
}
private async Task <string> FindBucketLocationAsync(string repositoryName, string branchName)
{
var request = new GetBucketLocationRequest()
{
BucketName = GetBucketName(repositoryName, branchName)
};
GetBucketLocationResponse response = await s3Client.GetBucketLocationAsync(request);
return(response.Location.ToString());
}
static string FindBucketLocation(IAmazonS3 client)
{
GetBucketLocationRequest request = new GetBucketLocationRequest()
{
BucketName = bucketName
};
GetBucketLocationResponse response = client.GetBucketLocation(request);
var bucketLocation = response.Location.ToString();
return(bucketLocation);
}
/// <summary>
/// Get the S3 bucket's region.
/// </summary>
/// <param name="client"></param>
/// <returns>The S3Region object representing the bucket region</returns>
private async Task <S3Region> GetBucketLocationAsync(IAmazonS3 client)
{
GetBucketLocationRequest request = new GetBucketLocationRequest
{
BucketName = Environment.GetEnvironmentVariable("BUCKET_NAME")
};
GetBucketLocationResponse response = await client.GetBucketLocationAsync(request);
return(response.Location);
}
private static async Task <string> FindBucketLocationAsync(IAmazonS3 client, string bucketName)
{
var request = new GetBucketLocationRequest()
{
BucketName = bucketName
};
GetBucketLocationResponse response = await client.GetBucketLocationAsync(request);
var bucketLocation = response.Location.ToString();
return(bucketLocation);
}
public async Task <string> FindBucketLocationAsync(string bucketName)
{
string bucketLocation;
var request = new GetBucketLocationRequest()
{
BucketName = bucketName
};
GetBucketLocationResponse response = await s3Client.GetBucketLocationAsync(request);
bucketLocation = response.Location.ToString();
return(bucketLocation);
}
private static async Task <string> FindBucketLocationAsync()
{
var request = new GetBucketLocationRequest()
{
BucketName = BUCKET_NAME
};
GetBucketLocationResponse response = await _s3Client.GetBucketLocationAsync(request);
string bucketLocation = response.Location.ToString();
return(bucketLocation);
}
public static RegionEndpoint GetBucketLocation(IAmazonS3 s3Client, string bucketName)
{
GetBucketLocationRequest getBucketLocationRequest = new GetBucketLocationRequest()
{
BucketName = bucketName
};
GetBucketLocationResponse getBucketLocationResponse = s3Client.GetBucketLocation(getBucketLocationRequest);
S3Region s3Region = getBucketLocationResponse.Location;
return(toRegionEndpoint(s3Region));
}
static async Task <string> FindBucketLocation(IAmazonS3 client)
{
string bucketLocation;
GetBucketLocationRequest request = new GetBucketLocationRequest()
{
BucketName = bucket
};
GetBucketLocationResponse response = await client.GetBucketLocationAsync(request);
bucketLocation = response.Location.ToString();
return(bucketLocation);
}
public static string GetBucketRegion(string bucketName)
{
try
{
AmazonS3Client s3 = new AmazonS3Client(Amazon.RegionEndpoint.USEast1);
GetBucketLocationResponse response = s3.GetBucketLocation(new GetBucketLocationRequest {
BucketName = bucketName
});
return(string.IsNullOrEmpty(response.Location.Value) ? Amazon.RegionEndpoint.USEast1.SystemName : response.Location.Value);
}
catch (Exception ex)
{
throw new ApplicationException("Unable to query region for bucket '" + bucketName + "'", ex);
}
}
static void GetBucketLocation()
{
try
{
GetBucketLocationRequest request = new GetBucketLocationRequest()
{
BucketName = bucketName
};
GetBucketLocationResponse response = client.GetBucketLocation(request);
Console.WriteLine("Get bucket location response: {0}", response.StatusCode);
Console.WriteLine("Bucket Location: {0}", response.Location);
}
catch (ObsException ex)
{
Console.WriteLine("Exception errorcode: {0}, when get bucket location.", ex.ErrorCode);
Console.WriteLine("Exception errormessage: {0}", ex.ErrorMessage);
}
}
private static void UnmarshallResult(XmlUnmarshallerContext context, GetBucketLocationResponse response)
{
int currentDepth = context.get_CurrentDepth();
int num = 1;
while (context.Read())
{
if (context.get_IsStartElement() || context.get_IsAttribute())
{
if (context.TestExpression("LocationConstraint", num))
{
response.Location = StringUnmarshaller.GetInstance().Unmarshall(context);
}
}
else if (context.get_IsEndElement() && context.get_CurrentDepth() < currentDepth)
{
break;
}
}
}
// Return a list of folders (buckets)
// Returns collection of CloudFolder objects, or null on error.
public override List <CloudFolder> ListFolders()
{
try
{
this.Exception = null;
List <CloudFolder> results = new List <CloudFolder>();
Task <ListBucketsResponse> task = StorageClient.ListBucketsAsync();
task.Wait();
ListBucketsResponse response = task.Result;
if (response != null && response.Buckets != null)
{
foreach (S3Bucket bucket in response.Buckets)
{
GetBucketLocationRequest GBLrequest = new GetBucketLocationRequest()
{
BucketName = bucket.BucketName
};
Task <GetBucketLocationResponse> task2 = StorageClient.GetBucketLocationAsync(GBLrequest);
task2.Wait();
GetBucketLocationResponse GBLresponse = task2.Result;
if (GBLresponse.Location == this.Region)
{
results.Add(new CloudFolder(bucket.BucketName, this.Account, bucket));
}
}
}
return(results);
}
catch (Exception ex)
{
this.Exception = ex;
if (!this.HandleErrors)
{
throw ex;
}
return(null);
}
}
public override bool VerifyConfiguration(out string errorString)
{
bool isValid = true;
errorString = string.Empty;
IAmazonS3 amazonS3Client = null;
GetBucketLocationResponse response = null;
try
{
Convert.FromBase64String(_configurationObject.SecretAccessKey);
}
catch (Exception ex)
{
errorString = string.Format("Invalid 'Secret Access Key': '{0}'", ex.Message);
isValid = false;
}
if (isValid)
{
try
{
amazonS3Client = new AmazonS3Client(
_configurationObject.AccessKeyId,
_configurationObject.SecretAccessKey,
_configurationObject.Region.ToRegionEndpoint()
);
response = amazonS3Client.GetBucketLocation(_configurationObject.BucketName);
}
catch (Exception ex)
{
errorString = ex.Message;
isValid = false;
}
}
return(isValid);
}
public ResultResponce getBucketLocation(string BucketName)
{
ResultResponce result = new ResultResponce();
try
{
GetBucketLocationResponse responce = s3Client.GetBucketLocation(new GetBucketLocationRequest()
{
BucketName = BucketName
});
var a = RegionEndpoint.GetBySystemName(responce.Location);
result.Error = false;
result.Data = responce.Location;
return(result);
}
catch (Exception ex)
{
result.Error = true;
result.ErrorMessage = ex.Message;
return(result);
}
}
public void TestGetBucketLocation()
{
GetBucketLocationResponse getBucketLocationResponse = this.client.GetBucketLocation(this.bucketName);
Assert.AreEqual(getBucketLocationResponse.LocationConstraint, "bj");
}
public void BucketSamples()
{
{
#region ListBuckets Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Issue call
ListBucketsResponse response = client.ListBuckets();
// View response data
Console.WriteLine("Buckets owner - {0}", response.Owner.DisplayName);
foreach (S3Bucket bucket in response.Buckets)
{
Console.WriteLine("Bucket {0}, Created on {1}", bucket.BucketName, bucket.CreationDate);
}
#endregion
}
{
#region BucketPolicy Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Put sample bucket policy (overwrite an existing policy)
string newPolicy = @"{
""Statement"":[{
""Sid"":""BasicPerms"",
""Effect"":""Allow"",
""Principal"": ""*"",
""Action"":[""s3:PutObject"",""s3:GetObject""],
""Resource"":[""arn:aws:s3:::samplebucketname/*""]
}]}";
PutBucketPolicyRequest putRequest = new PutBucketPolicyRequest
{
BucketName = "SampleBucket",
Policy = newPolicy
};
client.PutBucketPolicy(putRequest);
// Retrieve current policy
GetBucketPolicyRequest getRequest = new GetBucketPolicyRequest
{
BucketName = "SampleBucket"
};
string policy = client.GetBucketPolicy(getRequest).Policy;
Console.WriteLine(policy);
Debug.Assert(policy.Contains("BasicPerms"));
// Delete current policy
DeleteBucketPolicyRequest deleteRequest = new DeleteBucketPolicyRequest
{
BucketName = "SampleBucket"
};
client.DeleteBucketPolicy(deleteRequest);
// Retrieve current policy and verify that it is null
policy = client.GetBucketPolicy(getRequest).Policy;
Debug.Assert(policy == null);
#endregion
}
{
#region GetBucketLocation Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Construct request
GetBucketLocationRequest request = new GetBucketLocationRequest
{
BucketName = "SampleBucket"
};
// Issue call
GetBucketLocationResponse response = client.GetBucketLocation(request);
// View response data
Console.WriteLine("Bucket location - {0}", response.Location);
#endregion
}
{
#region PutBucket Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Construct request
PutBucketRequest request = new PutBucketRequest
{
BucketName = "SampleBucket",
BucketRegion = S3Region.EU, // set region to EU
CannedACL = S3CannedACL.PublicRead // make bucket publicly readable
};
// Issue call
PutBucketResponse response = client.PutBucket(request);
#endregion
}
{
#region DeleteBucket Sample 1
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Construct request
DeleteBucketRequest request = new DeleteBucketRequest
{
BucketName = "SampleBucket"
};
// Issue call
DeleteBucketResponse response = client.DeleteBucket(request);
#endregion
}
{
#region DeleteBucket Sample 2
// Create a client
AmazonS3Client client = new AmazonS3Client();
// List and delete all objects
ListObjectsRequest listRequest = new ListObjectsRequest
{
BucketName = "SampleBucket"
};
ListObjectsResponse listResponse;
do
{
// Get a list of objects
listResponse = client.ListObjects(listRequest);
foreach (S3Object obj in listResponse.S3Objects)
{
// Delete each object
client.DeleteObject(new DeleteObjectRequest
{
BucketName = "SampleBucket",
Key = obj.Key
});
}
// Set the marker property
listRequest.Marker = listResponse.NextMarker;
} while (listResponse.IsTruncated);
// Construct DeleteBucket request
DeleteBucketRequest request = new DeleteBucketRequest
{
BucketName = "SampleBucket"
};
// Issue call
DeleteBucketResponse response = client.DeleteBucket(request);
#endregion
}
{
#region LifecycleConfiguration Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Put sample lifecycle configuration (overwrite an existing configuration)
LifecycleConfiguration newConfiguration = new LifecycleConfiguration
{
Rules = new List <LifecycleRule>
{
// Rule to delete keys with prefix "Test-" after 5 days
new LifecycleRule
{
Prefix = "Test-",
Expiration = new LifecycleRuleExpiration {
Days = 5
}
},
// Rule to delete keys in subdirectory "Logs" after 2 days
new LifecycleRule
{
Prefix = "Logs/",
Expiration = new LifecycleRuleExpiration {
Days = 2
},
Id = "log-file-removal"
}
}
};
PutLifecycleConfigurationRequest putRequest = new PutLifecycleConfigurationRequest
{
BucketName = "SampleBucket",
Configuration = newConfiguration
};
client.PutLifecycleConfiguration(putRequest);
// Retrieve current configuration
GetLifecycleConfigurationRequest getRequest = new GetLifecycleConfigurationRequest
{
BucketName = "SampleBucket"
};
LifecycleConfiguration configuration = client.GetLifecycleConfiguration(getRequest).Configuration;
Console.WriteLine("Configuration contains {0} rules", configuration.Rules.Count);
foreach (LifecycleRule rule in configuration.Rules)
{
Console.WriteLine("Rule");
Console.WriteLine(" Prefix = " + rule.Prefix);
Console.WriteLine(" Expiration (days) = " + rule.Expiration.Days);
Console.WriteLine(" Id = " + rule.Id);
Console.WriteLine(" Status = " + rule.Status);
}
// Put a new configuration and overwrite the existing configuration
configuration.Rules.RemoveAt(0); // remove first rule
client.PutLifecycleConfiguration(putRequest);
// Delete current configuration
DeleteLifecycleConfigurationRequest deleteRequest = new DeleteLifecycleConfigurationRequest
{
BucketName = "SampleBucket"
};
client.DeleteLifecycleConfiguration(deleteRequest);
// Retrieve current configuration and verify that it is null
configuration = client.GetLifecycleConfiguration(getRequest).Configuration;
Debug.Assert(configuration == null);
#endregion
}
}
static void Main(string[] args)
{
startTime = DateTime.Now;
//Catch exceptions
AppDomain currentDomain = AppDomain.CurrentDomain;
currentDomain.UnhandledException += new UnhandledExceptionEventHandler(MyHandler);
//Catch ctrl+c to we can put out our summary
Console.CancelKeyPress += (sender, eventArgs) =>
{
eventArgs.Cancel = false;
WriteLog("!!CANCELLED!!");
keepRunning = false;
PrintSummary();
};
ServicePointManager.Expect100Continue = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3;
string strConfigFile = "CrossCloudBackup.xml";
if (args.Length > 0)
{
strConfigFile = args[0];
}
if (!File.Exists(strConfigFile))
{
new XDocument(
new XDeclaration("1.0", "utf-8", "yes"),
new XComment("CrossCloudBackup Local Config File"),
new XElement("root",
new XElement("AWSKey", "someValue"),
new XElement("AWSSecret", "someValue"),
new XElement("AWSRegion", "eu-west-1"),
new XElement("RSUsername", "someValue"),
new XElement("RSAPIKey", "someValue"),
new XElement("RSUseServiceNet", "false"),
new XElement("RSLocation", "UK"),
new XElement("ExcludeBuckets", ""),
new XElement("ExcludeContainers", ""),
//new XElement("MirrorAll", "true"), /TODO: Add Selective Sync
new XElement("RSBackupContainer", "s3-backup"),
new XElement("S3BackupBucket", "rs-backup")
//new XElement("TransferThreads", "3") //TODO: Add Threading
)
)
.Save(strConfigFile);
Console.WriteLine(strConfigFile + " not found, blank one created.");
Console.WriteLine("Press enter to exit...");
Console.ReadLine();
Environment.Exit(1);
}
//We know the config file exists, so open and read values
XDocument config = XDocument.Load(strConfigFile);
//Get AWS config
string AWSKey = config.Element("root").Element("AWSKey").Value;
string AWSSecret = config.Element("root").Element("AWSSecret").Value;
RegionEndpoint region = RegionEndpoint.EUWest1;
switch (config.Element("root").Element("AWSRegion").Value)
{
case "eu-west-1":
region = RegionEndpoint.EUWest1;
break;
case "sa-east-1":
region = RegionEndpoint.SAEast1;
break;
case "us-east-1":
region = RegionEndpoint.USEast1;
break;
case "ap-northeast-1":
region = RegionEndpoint.APNortheast1;
break;
case "us-west-2":
region = RegionEndpoint.USWest2;
break;
case "us-west-1":
region = RegionEndpoint.USWest1;
break;
case "ap-southeast-1":
region = RegionEndpoint.APSoutheast1;
break;
case "ap-southeast-2":
region = RegionEndpoint.APSoutheast2;
break;
default:
region = RegionEndpoint.EUWest1;
break;
}
//Create a connection to S3
WriteLog("Connecting to S3");
S3Client = AWSClientFactory.CreateAmazonS3Client(AWSKey, AWSSecret, region);
//Get RS config
Rackspace.CloudFiles.Utils.AuthUrl rsRegion = Rackspace.CloudFiles.Utils.AuthUrl.US;
switch (config.Element("root").Element("RSLocation").Value)
{
case "UK":
rsRegion = Rackspace.CloudFiles.Utils.AuthUrl.UK;
break;
case "US":
rsRegion = Rackspace.CloudFiles.Utils.AuthUrl.US;
break;
case "Mosso":
rsRegion = Rackspace.CloudFiles.Utils.AuthUrl.Mosso;
break;
}
//Create connection to Rackspace
WriteLog("Connecting to Rackspace Cloud Files");
RSConnection = new Connection(new UserCredentials(config.Element("root").Element("RSUsername").Value, config.Element("root").Element("RSAPIKey").Value, rsRegion), Convert.ToBoolean(config.Element("root").Element("RSUseServiceNet").Value));
//Get exclusions
string[] excludeBuckets = config.Element("root").Element("ExcludeBuckets").Value.Split(',');
string[] excludeContainers = config.Element("root").Element("ExcludeContainers").Value.Split(',');
//First process all the S3 buckets and stream right into Rackspace container.
WriteLog("Listing S3 Buckets");
ListBucketsResponse response = S3Client.ListBuckets();
WriteLog("Found " + response.Buckets.Count() + " buckets");
foreach (S3Bucket bucket in response.Buckets)
{
if (bucket.BucketName == config.Element("root").Element("S3BackupBucket").Value)
{
WriteLog("Skipping " + bucket.BucketName + " as backup folder");
}
else if (excludeBuckets.Contains(bucket.BucketName))
{
WriteLog("Skipping " + bucket.BucketName + " as in exclusions");
}
else
{
//We need to know if the bucket is in the right region, otherwise it will error
GetBucketLocationResponse locResponse = S3Client.GetBucketLocation(new GetBucketLocationRequest().WithBucketName(bucket.BucketName));
if (locResponse.Location == config.Element("root").Element("AWSRegion").Value)
{
WriteLog("Processing " + bucket.BucketName);
//Get list of files
ListObjectsRequest request = new ListObjectsRequest();
request.BucketName = bucket.BucketName;
do
{
ListObjectsResponse filesResponse = S3Client.ListObjects(request);
WriteLog("Found " + filesResponse.S3Objects.Count() + " files");
if (filesResponse.IsTruncated)
{
WriteLog("there are additional pages of files");
}
foreach (S3Object file in filesResponse.S3Objects)
{
bool bolTransfer = false;
//See if it exists on Rackspace
string uri = RSConnection.StorageUrl + "/" + config.Element("root").Element("RSBackupContainer").Value + "/" + bucket.BucketName + "/" + file.Key;
try
{
var req = (HttpWebRequest)WebRequest.Create(uri);
req.Headers.Add("X-Auth-Token", RSConnection.AuthToken);
req.Method = "HEAD";
//Compare Etags to see if we need to sync
using (var resp = req.GetResponse() as HttpWebResponse)
{
if ("\"" + resp.Headers["eTag"] + "\"" != file.ETag)
{
bolTransfer = true;
}
}
}
catch (System.Net.WebException e)
{
if (e.Status == WebExceptionStatus.ProtocolError && ((HttpWebResponse)e.Response).StatusCode == HttpStatusCode.NotFound)
{
//Item not found, so upload
bolTransfer = true;
}
//WriteLog("End Request to " + uri);
}
if (file.StorageClass == "GLACIER")
{
bolTransfer = false; //We can't get things out of Glacier, but they aer still listed here.
}
if (bolTransfer)
{
WriteLog("Syncing " + file.Key);
using (GetObjectResponse getResponse = S3Client.GetObject(new GetObjectRequest().WithBucketName(bucket.BucketName).WithKey(file.Key)))
{
using (Stream s = getResponse.ResponseStream)
{
//We can stream right from s3 to CF, no need to store in memory or filesystem.
var req = (HttpWebRequest)WebRequest.Create(uri);
req.Headers.Add("X-Auth-Token", RSConnection.AuthToken);
req.Method = "PUT";
req.SendChunked = true;
req.AllowWriteStreamBuffering = false;
req.Timeout = -1;
using (Stream stream = req.GetRequestStream())
{
byte[] data = new byte[8192];
int bytesRead = 0;
while ((bytesRead = s.Read(data, 0, data.Length)) > 0)
{
stream.Write(data, 0, bytesRead);
}
stream.Flush();
stream.Close();
}
req.GetResponse().Close();
}
}
intTransferred++;
bytesTransferred += file.Size;
}
else
{
WriteLog("Skipping " + file.Key);
intSkipped++;
}
//Check our exit condition
if (!keepRunning)
{
break;
}
}
//Loop if there is more than 1000 files
if (filesResponse.IsTruncated)
{
request.Marker = filesResponse.NextMarker;
}
else
{
request = null;
}
if (!keepRunning)
{
break;
}
} while (request != null);
}
}
if (!keepRunning)
{
break;
}
}
//Now get all the Rackspace containers and stream them to Amazon
WriteLog("Listing CF Containers");
List <string> lstContainers = RSConnection.GetContainers();
WriteLog("Found " + lstContainers.Count() + " containers");
foreach (string container in lstContainers)
{
if (container == config.Element("root").Element("RSBackupContainer").Value)
{
WriteLog("Skipping " + container + " as backup folder");
}
else if (excludeContainers.Contains(container))
{
WriteLog("Skipping " + container + " as in exclusions");
}
else
{
WriteLog("Processing " + container);
XmlDocument containerInfo = RSConnection.GetContainerInformationXml(container);
do
{
int filesCount = containerInfo.GetElementsByTagName("object").Count;
WriteLog("Found " + filesCount + " files");
foreach (XmlNode file in containerInfo.GetElementsByTagName("object"))
{
bool bolTransfer = false;
string strBucketName = config.Element("root").Element("S3BackupBucket").Value;
string strKey = container + file.SelectSingleNode("name").InnerText;
//See if the file exists on s3
try
{
GetObjectMetadataResponse metaResp = S3Client.GetObjectMetadata(new GetObjectMetadataRequest().WithBucketName(strBucketName).WithKey(strKey));
//Compare the etags
if (metaResp.ETag != "\"" + file.SelectSingleNode("hash").InnerText + "\"")
{
bolTransfer = true;
}
}
catch (Amazon.S3.AmazonS3Exception e)
{
bolTransfer = true;
}
if (bolTransfer)
{
WriteLog("Syncing " + file.SelectSingleNode("name").InnerText);
//God the C# binding sucks, so let's stream manually
string uri = RSConnection.StorageUrl + "/" + container + "/" + file.SelectSingleNode("name").InnerText;
var req = (HttpWebRequest)WebRequest.Create(uri);
req.Headers.Add("X-Auth-Token", RSConnection.AuthToken);
req.Method = "GET";
using (var resp = req.GetResponse() as HttpWebResponse)
{
using (Stream s = resp.GetResponseStream())
{
string today = String.Format("{0:ddd,' 'dd' 'MMM' 'yyyy' 'HH':'mm':'ss' 'zz00}", DateTime.Now);
string stringToSign = "PUT\n" +
"\n" +
file.SelectSingleNode("content_type").InnerText + "\n" +
"\n" +
"x-amz-date:" + today + "\n" +
"/" + strBucketName + "/" + strKey;
Encoding ae = new UTF8Encoding();
HMACSHA1 signature = new HMACSHA1(ae.GetBytes(AWSSecret));
string encodedCanonical = Convert.ToBase64String(signature.ComputeHash(ae.GetBytes(stringToSign)));
string authHeader = "AWS " + AWSKey + ":" + encodedCanonical;
string uriS3 = "https://" + strBucketName + ".s3.amazonaws.com/" + strKey;
var reqS3 = (HttpWebRequest)WebRequest.Create(uriS3);
reqS3.Headers.Add("Authorization", authHeader);
reqS3.Headers.Add("x-amz-date", today);
reqS3.ContentType = file.SelectSingleNode("content_type").InnerText;
reqS3.ContentLength = Convert.ToInt32(file.SelectSingleNode("bytes").InnerText);
reqS3.Method = "PUT";
reqS3.AllowWriteStreamBuffering = false;
if (reqS3.ContentLength == -1L)
{
reqS3.SendChunked = true;
}
using (Stream streamS3 = reqS3.GetRequestStream())
{
byte[] data = new byte[32768];
int bytesRead = 0;
while ((bytesRead = s.Read(data, 0, data.Length)) > 0)
{
streamS3.Write(data, 0, bytesRead);
}
streamS3.Flush();
streamS3.Close();
}
reqS3.GetResponse().Close();
}
}
intTransferred++;
bytesTransferred += Convert.ToInt64(file.SelectSingleNode("bytes").InnerText);
}
else
{
WriteLog("Skipping " + file.SelectSingleNode("name").InnerText);
intSkipped++;
}
//Check our exit condition
if (!keepRunning)
{
break;
}
}
if (filesCount < 10000)
{
containerInfo = null;
}
else
{
//Fetch the next list, but the Rackspace binding doesn't support markers with XML responses....
try
{
string uri = RSConnection.StorageUrl + "/" + container + "?format=xml&marker=" + Uri.EscapeUriString(containerInfo.FirstChild.NextSibling.LastChild.SelectSingleNode("name").InnerText);
var req = (HttpWebRequest)WebRequest.Create(uri);
req.Headers.Add("X-Auth-Token", RSConnection.AuthToken);
req.Method = "GET";
using (var resp = req.GetResponse() as HttpWebResponse)
{
using (var reader = new System.IO.StreamReader(resp.GetResponseStream(), ASCIIEncoding.ASCII))
{
string responseText = reader.ReadToEnd();
containerInfo.LoadXml(responseText);
}
}
}
catch (System.Net.WebException e)
{
if (e.Status == WebExceptionStatus.ProtocolError && ((HttpWebResponse)e.Response).StatusCode == HttpStatusCode.NotFound)
{
containerInfo = null;
}
}
}
} while (containerInfo != null);
}
}
if (keepRunning)
{
WriteLog("Completed");
PrintSummary();
}
}