/// <summary>
/// Create a <see cref="S3ToAzureTransferJob"/> representing a download-to-local-and-upload copy from one S3 object to Azure blob.
/// </summary>
/// <param name="sourceObject">S3 object used to create the job.</param>
/// <returns>A job representing a download-to-local-and-upload copy from one S3 object to Azure blob.</returns>
private static S3ToAzureTransferJob CreateTransferJob(S3Object sourceObject)
{
// Download the source object to a temporary file
GetObjectRequest getObjectRequest = new GetObjectRequest()
{
BucketName = S3BucketName,
Key = sourceObject.Key,
};
using (GetObjectResponse getObjectResponse = s3Client.GetObject(getObjectRequest))
{
string tempFile = Path.Combine(tempFolder, Guid.NewGuid().ToString());
getObjectResponse.WriteResponseStreamToFile(tempFile);
S3ToAzureTransferJob job = new S3ToAzureTransferJob()
{
Source = tempFile,
Name = sourceObject.Key,
ServiceSideCopy = false
};
return(job);
}
}
/// <summary>
/// List all objects from your S3 bucket and add one job into jobQueue for each listed object.
/// </summary>
private static void ListFromS3()
{
string previousMarker = null;
bool listFinish = false;
while (!listFinish)
{
ListObjectsRequest listObjectRequest = new ListObjectsRequest()
{
BucketName = S3BucketName,
Marker = previousMarker,
};
ListObjectsResponse listObjectResponse = s3Client.ListObjects(listObjectRequest);
previousMarker = listObjectResponse.NextMarker;
listFinish = String.IsNullOrEmpty(previousMarker);
foreach (var source in listObjectResponse.S3Objects)
{
ConsoleWriteLine("Object listed from bucket {0}: {1}", S3BucketName, source.Key);
// By default, the sample will download an amazon s3 object into a local file and
// then upload it into Microsoft Azure Stroage with DataMovement library later.
S3ToAzureTransferJob job = CreateTransferJob(source);
// You can choose to use azure server side copy by replacing CreateTransferJob above
// with the following statement. When server side copy is used, Azure server will copy
// the data directly from the URI provided and no data is downloaded to local.
// TransferJob job = CreateServiceSideTransferJob(source);
jobQueue.Add(job);
}
}
jobQueue.CompleteAdding();
}
/// <summary>
/// Get job from jobQueue and transfer data into your Azure blob container.
/// </summary>
private static void TransferToAzure()
{
// Create the container if it doesn't exist yet
CloudBlobClient client = azureClient;
CloudBlobContainer container = client.GetContainerReference(AzureContainerName);
container.CreateIfNotExists();
SingleTransferContext context = new SingleTransferContext();
// Add progress handler
context.ProgressHandler = progressRecorder;
context.ShouldOverwriteCallbackAsync = Program.OverwritePrompt;
while (!jobQueue.IsCompleted)
{
S3ToAzureTransferJob job = null;
try
{
job = jobQueue.Take();
}
catch (InvalidOperationException)
{
// No more jobs to do
}
if (job == null)
{
break;
}
countdownEvent.AddCount();
CloudBlockBlob cloudBlob = container.GetBlockBlobReference(job.Name);
ConsoleWriteLine("Start to transfer {0} to azure.", job.Name);
Task task = null;
try
{
if (!job.ServiceSideCopy)
{
// By default, the sample will download an amazon s3 object into a local file and
// then upload it into Microsoft Azure Stroage with DataMovement library.
task = TransferManager.UploadAsync(job.Source, cloudBlob, null, context);
}
else
{
// When server side copy is used, Azure server will copy the data directly from the URI
// provided and no data is downloaded to local.
task = TransferManager.CopyAsync(new Uri(job.Source), cloudBlob, true, null, context);
}
}
catch (Exception e)
{
ConsoleWriteLine("Error occurs when transferring {0}: {1}", job.Name, e.ToString());
}
if (task != null)
{
task.ContinueWith(t =>
{
if (t.IsFaulted)
{
ConsoleWriteLine("Error occurs when transferring {0}: {1}", job.Name, t.Exception.ToString());
}
else
{
ConsoleWriteLine("Succeed to transfer data to blob {0}", job.Name);
}
// Signal the countdown event when one transfer job finishes.
countdownEvent.Signal();
});
}
else
{
// Signal the countdown event when one transfer job finishes.
countdownEvent.Signal();
}
}
// Signal the countdown event to unblock the main thread when all data are transferred.
countdownEvent.Signal();
}