private static void WriteToBlobStorage(BucketBlobPackage package) { foreach (var item in package.FileInfo) { using (client = new AmazonS3Client(package.AccessKey, package.SecretAccessKey, GetAwsRegionFromString(package.BucketInfo.Region))) { GetObjectRequest request = new GetObjectRequest { BucketName = package.BucketInfo.Name, Key = item.Key }; using (GetObjectResponse response = client.GetObject(request)) { using (Stream responseStream = response.ResponseStream) { string fileExtension = item.Key.Substring(Math.Max(0, item.Key.Length - 4)).ToUpper(); switch (fileExtension) { case ".ZIP": CopyZIPtoBlob(responseStream); break; default: S3Object o = item.Value as S3Object; CopySingleFileToBlob(responseStream, item.Key, o.Size); break; } } } } } }
// Please set the following connection strings in app.config for this WebJob to run: // AzureWebJobsDashboard and AzureWebJobsStorage private static void Main() { #region Parse Configuration Settings AwsSettings awsSettings = new AwsSettings(); string AWS_SETTINGS = CloudConfigurationManager.GetSetting("AWS_SETTINGS"); awsSettings = JsonConvert.DeserializeObject <AwsSettings>(AWS_SETTINGS); azureStorageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("AzureStorageConnectionString")); azureStorageContainerName = CloudConfigurationManager.GetSetting("AzureStorageContainerName"); string runMode = CloudConfigurationManager.GetSetting("runMode"); if (runMode == "INTERACTIVE") { interactiveMode = true; } #endregion Console.WriteLine("AWSBillingWebJob: {0}.", awsSettings.Description); azureClient = azureStorageAccount.CreateCloudBlobClient(); azureClient.GetContainerReference(azureStorageContainerName).CreateIfNotExists(); // For each account in AwsSettings foreach (var account in awsSettings.Accounts) { ConsoleColor currentColor = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine(@"Processing AWS Account {0}", account); Console.ForegroundColor = currentColor; // For each bucket we need to inspect for this account foreach (var bucket in account.Buckets) { Console.WriteLine(@"Processing Bucket {0}", bucket.Name); count = 0; files = new Dictionary <string, S3Object>(); Amazon.RegionEndpoint region = GetAwsRegionFromString(bucket.Region); using (client = new AmazonS3Client(account.AccessKey, account.SecretAccessKey, region)) { try { ListObjectsV2Request request = new ListObjectsV2Request { BucketName = bucket.Name, MaxKeys = 10, Prefix = account.AccountId }; ListObjectsV2Response response; do { response = client.ListObjectsV2(request); // Process response. foreach (S3Object entry in response.S3Objects) { if (FileIsRequired(entry.Key, bucket.Patterns)) { count++; //Console.WriteLine("key = {0} size = {1}, Date={2}", entry.Key, entry.Size, entry.LastModified); S3Object theFile = new S3Object(); theFile.BucketName = entry.BucketName; theFile.ETag = entry.ETag; theFile.LastModified = entry.LastModified; theFile.Owner = entry.Owner; theFile.Size = entry.Size; theFile.StorageClass = entry.StorageClass; files.Add(entry.Key, theFile); } } request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated == true); } catch (AmazonS3Exception amazonS3Exception) { if (amazonS3Exception.ErrorCode != null && (amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") || amazonS3Exception.ErrorCode.Equals("InvalidSecurity"))) { Console.WriteLine("Check the provided AWS Credentials."); Console.WriteLine( "To sign up for service, go to http://aws.amazon.com/s3"); } else { Console.WriteLine( "Error occurred. Message:'{0}' when listing objects", amazonS3Exception.Message); } throw amazonS3Exception; } } ConsoleColor savedColour = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("Bucket{0} contains {1} files of interest:", bucket.Name, count); Console.ForegroundColor = savedColour; BucketBlobPackage package = new BucketBlobPackage(); package.AccountId = account.AccountId; package.AccessKey = account.AccessKey; package.SecretAccessKey = account.SecretAccessKey; package.BucketInfo = bucket; package.FileInfo = files; WriteToBlobStorage(package); } } Console.WriteLine("AWSBillingToAzureBlob job has completed sucessfully"); if (interactiveMode) { Console.ReadKey(); } }