/// <summary>
/// Create a <see cref="S3ToAzureTransferJob"/> representing a download-to-local-and-upload copy from one S3 object to Azure blob.
/// </summary>
/// <param name="sourceObject">S3 object used to create the job.</param>
/// <returns>A job representing a download-to-local-and-upload copy from one S3 object to Azure blob.</returns>
private static AliyunToAzureTransferJob CreateTransferJob(OssObjectSummary objectSummary)
{
// Download the source object to a temporary file
GetObjectRequest getObjectRequest = new GetObjectRequest(PocConfig.SourceBucket, objectSummary.Key);
using (OssObject ossObject = ossClient.GetObject(getObjectRequest))
{
string tempFile = Path.Combine(tempFolder, Guid.NewGuid().ToString());
string md5;
string sha256;
string relativeName = PocUtil.GetRelativeName(ossObject.Key, PocConfig.SourceDir);
// Check if the destination blob already exists.
CloudBlobContainer container = azureClient.GetContainerReference(PocConfig.DestContainer);
CloudBlockBlob destBlob = container.GetBlockBlobReference(PocUtil.GetFullName(relativeName, PocConfig.DestDir));
try
{
if (destBlob.Exists(PocConstant.DefaultBlobRequestOptions))
{
Console.WriteLine("Dest already exist {0}", destBlob.Uri.ToString());
pocManifest.AddFailed(relativeName, PocErrorString.DestAlreadyExist);
return(null);
}
PocUtil.DownloadOssObject2File(ossObject, tempFile, out md5, out sha256);
}
catch (Exception e)
{
Console.Error.WriteLine("Fail to download from aliyun {0}. Error: {1}", objectSummary.Key, e.ToString());
pocManifest.AddFailed(relativeName, PocErrorString.DownloadFailed);
return(null);
}
if (!VerifyDownloadSHA(relativeName, sha256))
{
Console.Error.WriteLine("Download content miss match {0}. SHA: {1}", objectSummary.Key, sha256);
pocManifest.AddFailed(relativeName, PocErrorString.DownloadContentMissMatch);
return(null);
}
AliyunToAzureTransferJob job = new AliyunToAzureTransferJob()
{
Source = tempFile,
Name = relativeName,
ContentMD5 = md5,
};
return(job);
}
}
static void ListFromAliyun()
{
string marker = markerJournal.Marker;
int listedCount = 0;
long totalSize = 0;
do
{
markerJournal.Update(marker);
string actualPrefix = String.IsNullOrEmpty(PocConfig.SourceDir) ? PocConfig.Prefix : PocConfig.SourceDir + "/" + PocConfig.Prefix;
ListObjectsRequest listObjectsReq = new ListObjectsRequest(PocConfig.SourceBucket)
{
MaxKeys = PocConfig.ListStep,
Marker = marker,
Prefix = actualPrefix,
};
ObjectListing listedObjects = ossClient.ListObjects(listObjectsReq);
marker = listedObjects.NextMarker;
foreach (OssObjectSummary objectSummary in listedObjects.ObjectSummaries)
{
Console.WriteLine("{0} (size:{1}; LMT:{2})", objectSummary.Key, objectSummary.Size, objectSummary.LastModified);
totalSize += objectSummary.Size;
AliyunToAzureTransferJob job = CreateTransferJob(objectSummary);
if (++listedCount > PocConfig.MaxFileNumber)
{
// goto Finish;
}
if (job == null)
{
continue;
}
jobQueue.Add(job);
}
}while (!string.IsNullOrEmpty(marker));
Finish:
jobQueue.CompleteAdding();
Console.WriteLine("List complete");
}
/// <summary>
/// Get job from jobQueue and transfer data into your Azure blob container.
/// </summary>
private static void TransferToAzure()
{
// Create the container if it doesn't exist yet
CloudBlobClient client = azureClient;
CloudBlobContainer container = client.GetContainerReference(PocConfig.DestContainer);
container.CreateIfNotExists();
while (!jobQueue.IsCompleted)
{
AliyunToAzureTransferJob job = null;
try
{
job = jobQueue.Take();
}
catch (InvalidOperationException)
{
// No more jobs to do
}
if (job == null)
{
break;
}
countdownEvent.AddCount();
CloudBlockBlob cloudBlob = container.GetBlockBlobReference(PocUtil.GetFullName(job.Name, PocConfig.DestDir));
Console.WriteLine("Start to transfer {0} to azure.", job.Name);
Task task = null;
try
{
TransferContext context = new TransferContext();
context.OverwriteCallback = (source, dest) =>
{
Console.WriteLine("Dest already exist {0}", dest);
pocManifest.AddFailed(job.Name, PocErrorString.DestAlreadyExist);
return(false);
};
// By default, the sample will download an amazon s3 object into a local file and
// then upload it into Microsoft Azure Stroage with DataMovement library.
task = TransferManager.UploadAsync(job.Source, cloudBlob, null, context);
}
catch (Exception e)
{
Console.WriteLine("Error occurs when transferring {0}: {1}", job.Name, e.ToString());
}
if (task != null)
{
task.ContinueWith(t =>
{
if (t.IsFaulted)
{
if (t.Exception == null || t.Exception.InnerException == null || !(t.Exception.InnerException is TransferException) ||
(t.Exception.InnerException as TransferException).ErrorCode != TransferErrorCode.NotOverwriteExistingDestination)
{
pocManifest.AddFailed(job.Name, PocErrorString.UploadFailed);
Console.Error.WriteLine("Error occurs when transferring {0}: {1}", job.Name, t.Exception.ToString());
}
}
else
{
cloudBlob.FetchAttributes(options: PocConstant.DefaultBlobRequestOptions);
if (job.ContentMD5 != cloudBlob.Properties.ContentMD5)
{
pocManifest.AddFailed(job.Name, PocErrorString.UploadContentMissMatch);
Console.Error.WriteLine("Data loss! {0}", job.Name);
}
else
{
pocManifest.AddTransferred(job.Name);
// TO-DO, delete successfully transfered temp file
if (File.Exists(job.Source))
{
FileInfo fi = new FileInfo(job.Source);
if (fi.Attributes.ToString().IndexOf("ReadOnly") != -1)
{
fi.Attributes = FileAttributes.Normal;
}
File.Delete(job.Source);
}
}
Console.WriteLine("Source md5: {0}, Destination md5: {1}. Succeed to transfer data to blob {2}", job.ContentMD5, cloudBlob.Properties.ContentMD5, job.Name);
}
// Signal the countdown event when one transfer job finishes.
countdownEvent.Signal();
});
}
else
{
// Signal the countdown event when one transfer job finishes.
countdownEvent.Signal();
}
}
// Signal the countdown event to unblock the main thread when all data are transferred.
countdownEvent.Signal();
}