private static void WriteToBlobStorage(BucketBlobPackage package)
{
foreach (var item in package.FileInfo)
{
using (client = new AmazonS3Client(package.AccessKey, package.SecretAccessKey, GetAwsRegionFromString(package.BucketInfo.Region)))
{
GetObjectRequest request = new GetObjectRequest
{
BucketName = package.BucketInfo.Name,
Key = item.Key
};
using (GetObjectResponse response = client.GetObject(request))
{
using (Stream responseStream = response.ResponseStream)
{
string fileExtension = item.Key.Substring(Math.Max(0, item.Key.Length - 4)).ToUpper();
switch (fileExtension)
{
case ".ZIP":
CopyZIPtoBlob(responseStream);
break;
default:
S3Object o = item.Value as S3Object;
CopySingleFileToBlob(responseStream, item.Key, o.Size);
break;
}
}
}
}
}
}
// Please set the following connection strings in app.config for this WebJob to run:
// AzureWebJobsDashboard and AzureWebJobsStorage
private static void Main()
{
#region Parse Configuration Settings
AwsSettings awsSettings = new AwsSettings();
string AWS_SETTINGS = CloudConfigurationManager.GetSetting("AWS_SETTINGS");
awsSettings = JsonConvert.DeserializeObject <AwsSettings>(AWS_SETTINGS);
azureStorageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("AzureStorageConnectionString"));
azureStorageContainerName = CloudConfigurationManager.GetSetting("AzureStorageContainerName");
string runMode = CloudConfigurationManager.GetSetting("runMode");
if (runMode == "INTERACTIVE")
{
interactiveMode = true;
}
#endregion
Console.WriteLine("AWSBillingWebJob: {0}.", awsSettings.Description);
azureClient = azureStorageAccount.CreateCloudBlobClient();
azureClient.GetContainerReference(azureStorageContainerName).CreateIfNotExists();
// For each account in AwsSettings
foreach (var account in awsSettings.Accounts)
{
ConsoleColor currentColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine(@"Processing AWS Account {0}", account);
Console.ForegroundColor = currentColor;
// For each bucket we need to inspect for this account
foreach (var bucket in account.Buckets)
{
Console.WriteLine(@"Processing Bucket {0}", bucket.Name);
count = 0;
files = new Dictionary <string, S3Object>();
Amazon.RegionEndpoint region = GetAwsRegionFromString(bucket.Region);
using (client = new AmazonS3Client(account.AccessKey, account.SecretAccessKey, region))
{
try
{
ListObjectsV2Request request = new ListObjectsV2Request
{
BucketName = bucket.Name,
MaxKeys = 10,
Prefix = account.AccountId
};
ListObjectsV2Response response;
do
{
response = client.ListObjectsV2(request);
// Process response.
foreach (S3Object entry in response.S3Objects)
{
if (FileIsRequired(entry.Key, bucket.Patterns))
{
count++;
//Console.WriteLine("key = {0} size = {1}, Date={2}", entry.Key, entry.Size, entry.LastModified);
S3Object theFile = new S3Object();
theFile.BucketName = entry.BucketName;
theFile.ETag = entry.ETag;
theFile.LastModified = entry.LastModified;
theFile.Owner = entry.Owner;
theFile.Size = entry.Size;
theFile.StorageClass = entry.StorageClass;
files.Add(entry.Key, theFile);
}
}
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated == true);
}
catch (AmazonS3Exception amazonS3Exception)
{
if (amazonS3Exception.ErrorCode != null &&
(amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId")
||
amazonS3Exception.ErrorCode.Equals("InvalidSecurity")))
{
Console.WriteLine("Check the provided AWS Credentials.");
Console.WriteLine(
"To sign up for service, go to http://aws.amazon.com/s3");
}
else
{
Console.WriteLine(
"Error occurred. Message:'{0}' when listing objects",
amazonS3Exception.Message);
}
throw amazonS3Exception;
}
}
ConsoleColor savedColour = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Bucket{0} contains {1} files of interest:",
bucket.Name, count);
Console.ForegroundColor = savedColour;
BucketBlobPackage package = new BucketBlobPackage();
package.AccountId = account.AccountId;
package.AccessKey = account.AccessKey;
package.SecretAccessKey = account.SecretAccessKey;
package.BucketInfo = bucket;
package.FileInfo = files;
WriteToBlobStorage(package);
}
}
Console.WriteLine("AWSBillingToAzureBlob job has completed sucessfully");
if (interactiveMode)
{
Console.ReadKey();
}
}