コード例 #1
0
        /// <summary>
        /// Create a <see cref="S3ToAzureTransferJob"/> representing a download-to-local-and-upload copy from one S3 object to Azure blob.
        /// </summary>
        /// <param name="sourceObject">S3 object used to create the job.</param>
        /// <returns>A job representing a download-to-local-and-upload copy from one S3 object to Azure blob.</returns>
        private static AliyunToAzureTransferJob CreateTransferJob(OssObjectSummary objectSummary)
        {
            // Download the source object to a temporary file
            GetObjectRequest getObjectRequest = new GetObjectRequest(PocConfig.SourceBucket, objectSummary.Key);



            using (OssObject ossObject = ossClient.GetObject(getObjectRequest))
            {
                string tempFile = Path.Combine(tempFolder, Guid.NewGuid().ToString());
                string md5;
                string sha256;
                string relativeName = PocUtil.GetRelativeName(ossObject.Key, PocConfig.SourceDir);

                // Check if the destination blob already exists.
                CloudBlobContainer container = azureClient.GetContainerReference(PocConfig.DestContainer);
                CloudBlockBlob     destBlob  = container.GetBlockBlobReference(PocUtil.GetFullName(relativeName, PocConfig.DestDir));

                try
                {
                    if (destBlob.Exists(PocConstant.DefaultBlobRequestOptions))
                    {
                        Console.WriteLine("Dest already exist {0}", destBlob.Uri.ToString());
                        pocManifest.AddFailed(relativeName, PocErrorString.DestAlreadyExist);
                        return(null);
                    }

                    PocUtil.DownloadOssObject2File(ossObject, tempFile, out md5, out sha256);
                }
                catch (Exception e)
                {
                    Console.Error.WriteLine("Fail to download from aliyun {0}. Error: {1}", objectSummary.Key, e.ToString());
                    pocManifest.AddFailed(relativeName, PocErrorString.DownloadFailed);
                    return(null);
                }

                if (!VerifyDownloadSHA(relativeName, sha256))
                {
                    Console.Error.WriteLine("Download content miss match {0}. SHA: {1}", objectSummary.Key, sha256);
                    pocManifest.AddFailed(relativeName, PocErrorString.DownloadContentMissMatch);
                    return(null);
                }

                AliyunToAzureTransferJob job = new AliyunToAzureTransferJob()
                {
                    Source     = tempFile,
                    Name       = relativeName,
                    ContentMD5 = md5,
                };

                return(job);
            }
        }
コード例 #2
0
        /// <summary>
        /// Get job from jobQueue and transfer data into your Azure blob container.
        /// </summary>
        private static void TransferToAzure()
        {
            // Create the container if it doesn't exist yet
            CloudBlobClient    client    = azureClient;
            CloudBlobContainer container = client.GetContainerReference(PocConfig.DestContainer);

            container.CreateIfNotExists();

            while (!jobQueue.IsCompleted)
            {
                AliyunToAzureTransferJob job = null;
                try
                {
                    job = jobQueue.Take();
                }
                catch (InvalidOperationException)
                {
                    // No more jobs to do
                }

                if (job == null)
                {
                    break;
                }

                countdownEvent.AddCount();

                CloudBlockBlob cloudBlob = container.GetBlockBlobReference(PocUtil.GetFullName(job.Name, PocConfig.DestDir));

                Console.WriteLine("Start to transfer {0} to azure.", job.Name);

                Task task = null;

                try
                {
                    TransferContext context = new TransferContext();

                    context.OverwriteCallback = (source, dest) =>
                    {
                        Console.WriteLine("Dest already exist {0}", dest);
                        pocManifest.AddFailed(job.Name, PocErrorString.DestAlreadyExist);
                        return(false);
                    };

                    // By default, the sample will download an amazon s3 object into a local file and
                    // then upload it into Microsoft Azure Stroage with DataMovement library.
                    task = TransferManager.UploadAsync(job.Source, cloudBlob, null, context);
                }
                catch (Exception e)
                {
                    Console.WriteLine("Error occurs when transferring {0}: {1}", job.Name, e.ToString());
                }

                if (task != null)
                {
                    task.ContinueWith(t =>
                    {
                        if (t.IsFaulted)
                        {
                            if (t.Exception == null || t.Exception.InnerException == null || !(t.Exception.InnerException is TransferException) ||
                                (t.Exception.InnerException as TransferException).ErrorCode != TransferErrorCode.NotOverwriteExistingDestination)
                            {
                                pocManifest.AddFailed(job.Name, PocErrorString.UploadFailed);
                                Console.Error.WriteLine("Error occurs when transferring {0}: {1}", job.Name, t.Exception.ToString());
                            }
                        }
                        else
                        {
                            cloudBlob.FetchAttributes(options: PocConstant.DefaultBlobRequestOptions);
                            if (job.ContentMD5 != cloudBlob.Properties.ContentMD5)
                            {
                                pocManifest.AddFailed(job.Name, PocErrorString.UploadContentMissMatch);
                                Console.Error.WriteLine("Data loss! {0}", job.Name);
                            }
                            else
                            {
                                pocManifest.AddTransferred(job.Name);

                                // TO-DO, delete successfully transfered temp file
                                if (File.Exists(job.Source))
                                {
                                    FileInfo fi = new FileInfo(job.Source);
                                    if (fi.Attributes.ToString().IndexOf("ReadOnly") != -1)
                                    {
                                        fi.Attributes = FileAttributes.Normal;
                                    }
                                    File.Delete(job.Source);
                                }
                            }

                            Console.WriteLine("Source md5: {0}, Destination md5: {1}. Succeed to transfer data to blob {2}", job.ContentMD5, cloudBlob.Properties.ContentMD5, job.Name);
                        }

                        // Signal the countdown event when one transfer job finishes.
                        countdownEvent.Signal();
                    });
                }
                else
                {
                    // Signal the countdown event when one transfer job finishes.
                    countdownEvent.Signal();
                }
            }

            // Signal the countdown event to unblock the main thread when all data are transferred.
            countdownEvent.Signal();
        }