/// <summary> /// Create a <see cref="S3ToAzureTransferJob"/> representing a download-to-local-and-upload copy from one S3 object to Azure blob. /// </summary> /// <param name="sourceObject">S3 object used to create the job.</param> /// <returns>A job representing a download-to-local-and-upload copy from one S3 object to Azure blob.</returns> private static S3ToAzureTransferJob CreateTransferJob(S3Object sourceObject) { // Download the source object to a temporary file GetObjectRequest getObjectRequest = new GetObjectRequest() { BucketName = S3BucketName, Key = sourceObject.Key, }; using (GetObjectResponse getObjectResponse = s3Client.GetObject(getObjectRequest)) { string tempFile = Path.Combine(tempFolder, Guid.NewGuid().ToString()); getObjectResponse.WriteResponseStreamToFile(tempFile); S3ToAzureTransferJob job = new S3ToAzureTransferJob() { Source = tempFile, Name = sourceObject.Key, ServiceSideCopy = false }; return(job); } }
/// <summary> /// List all objects from your S3 bucket and add one job into jobQueue for each listed object. /// </summary> private static void ListFromS3() { string previousMarker = null; bool listFinish = false; while (!listFinish) { ListObjectsRequest listObjectRequest = new ListObjectsRequest() { BucketName = S3BucketName, Marker = previousMarker, }; ListObjectsResponse listObjectResponse = s3Client.ListObjects(listObjectRequest); previousMarker = listObjectResponse.NextMarker; listFinish = String.IsNullOrEmpty(previousMarker); foreach (var source in listObjectResponse.S3Objects) { ConsoleWriteLine("Object listed from bucket {0}: {1}", S3BucketName, source.Key); // By default, the sample will download an amazon s3 object into a local file and // then upload it into Microsoft Azure Stroage with DataMovement library later. S3ToAzureTransferJob job = CreateTransferJob(source); // You can choose to use azure server side copy by replacing CreateTransferJob above // with the following statement. When server side copy is used, Azure server will copy // the data directly from the URI provided and no data is downloaded to local. // TransferJob job = CreateServiceSideTransferJob(source); jobQueue.Add(job); } } jobQueue.CompleteAdding(); }
/// <summary> /// Get job from jobQueue and transfer data into your Azure blob container. /// </summary> private static void TransferToAzure() { // Create the container if it doesn't exist yet CloudBlobClient client = azureClient; CloudBlobContainer container = client.GetContainerReference(AzureContainerName); container.CreateIfNotExists(); SingleTransferContext context = new SingleTransferContext(); // Add progress handler context.ProgressHandler = progressRecorder; context.ShouldOverwriteCallbackAsync = Program.OverwritePrompt; while (!jobQueue.IsCompleted) { S3ToAzureTransferJob job = null; try { job = jobQueue.Take(); } catch (InvalidOperationException) { // No more jobs to do } if (job == null) { break; } countdownEvent.AddCount(); CloudBlockBlob cloudBlob = container.GetBlockBlobReference(job.Name); ConsoleWriteLine("Start to transfer {0} to azure.", job.Name); Task task = null; try { if (!job.ServiceSideCopy) { // By default, the sample will download an amazon s3 object into a local file and // then upload it into Microsoft Azure Stroage with DataMovement library. task = TransferManager.UploadAsync(job.Source, cloudBlob, null, context); } else { // When server side copy is used, Azure server will copy the data directly from the URI // provided and no data is downloaded to local. task = TransferManager.CopyAsync(new Uri(job.Source), cloudBlob, true, null, context); } } catch (Exception e) { ConsoleWriteLine("Error occurs when transferring {0}: {1}", job.Name, e.ToString()); } if (task != null) { task.ContinueWith(t => { if (t.IsFaulted) { ConsoleWriteLine("Error occurs when transferring {0}: {1}", job.Name, t.Exception.ToString()); } else { ConsoleWriteLine("Succeed to transfer data to blob {0}", job.Name); } // Signal the countdown event when one transfer job finishes. countdownEvent.Signal(); }); } else { // Signal the countdown event when one transfer job finishes. countdownEvent.Signal(); } } // Signal the countdown event to unblock the main thread when all data are transferred. countdownEvent.Signal(); }