public void GetBlobCopyStateTest() { CloudBlobUtil blobUtil = new CloudBlobUtil(CommonStorageAccount); blobUtil.SetupTestContainerAndBlob(); StorageBlob.ICloudBlob destBlob = CopyBlobAndWaitForComplete(blobUtil); try { Test.Assert(destBlob.CopyState.Status == StorageBlob.CopyStatus.Success, String.Format("The blob copy using storage client should be success, actually it's {0}", destBlob.CopyState.Status)); PowerShellAgent agent = new PowerShellAgent(); Test.Assert(agent.GetAzureStorageBlobCopyState(blobUtil.ContainerName, destBlob.Name, false), "Get copy state should be success"); int expectedStateCount = 1; Test.Assert(agent.Output.Count == expectedStateCount, String.Format("Expected to get {0} copy state, actually it's {1}", expectedStateCount, agent.Output.Count)); StorageBlob.CopyStatus copyStatus = (StorageBlob.CopyStatus)agent.Output[0]["Status"]; Test.Assert(copyStatus == StorageBlob.CopyStatus.Success, String.Format("The blob copy should be success, actually it's {0}", copyStatus)); Uri sourceUri = (Uri)agent.Output[0]["Source"]; string expectedUri = CloudBlobUtil.ConvertCopySourceUri(blobUtil.Blob.Uri.ToString()); Test.Assert(sourceUri.ToString() == expectedUri, String.Format("Expected source uri is {0}, actully it's {1}", expectedUri, sourceUri.ToString())); Test.Assert(!agent.GetAzureStorageBlobCopyState(blobUtil.ContainerName, blobUtil.BlobName, false), "Get copy state should be fail since the specified blob don't have any copy operation"); Test.Assert(agent.ErrorMessages.Count > 0, "Should return error message"); string errorMessage = "Can not find copy task on specified blob"; Test.Assert(agent.ErrorMessages[0].StartsWith(errorMessage), String.Format("Error message should start with {0}, and actually it's {1}", errorMessage, agent.ErrorMessages[0])); } finally { blobUtil.CleanupTestContainerAndBlob(); } }
private void AssertFinishedCopyState(Uri SourceUri, int startIndex = 0) { string expectedSourceUri = CloudBlobUtil.ConvertCopySourceUri(SourceUri.ToString()); Test.Assert(agent.Output.Count > startIndex, String.Format("Should contain the great than {0} copy state, and actually it's {1}", startIndex, agent.Output.Count)); string sourceUri = ((Uri)agent.Output[startIndex]["Source"]).ToString(); Test.Assert(sourceUri.StartsWith(expectedSourceUri), String.Format("source uri should start with {0}, and actualy it's {1}", expectedSourceUri, sourceUri)); StorageBlob.CopyStatus status = (StorageBlob.CopyStatus)agent.Output[startIndex]["Status"]; Test.Assert(status != StorageBlob.CopyStatus.Pending, String.Format("Copy status should not be Pending, actually it's {0}", status)); string copyId = (string)agent.Output[startIndex]["CopyId"]; Test.Assert(!String.IsNullOrEmpty(copyId), "Copy ID should be not empty"); }
static void Main(string[] args) { // Create a reference of Windows Azure Storage Account StorageCredentials azureCloudStorageCredentials = new StorageCredentials(azureStorageAccountName, azureStorageAccountKey); CloudStorageAccount azureCloudStorageAccount = new CloudStorageAccount(azureCloudStorageCredentials, true); // Get a reference of Blob Container where the objects will be copied var blobContainer = azureCloudStorageAccount.CreateCloudBlobClient().GetContainerReference(azureBlobContainerName); // Create blob container if needed Console.WriteLine("Searching for the \"" + azureBlobContainerName + "\" blob container..."); if (blobContainer.CreateIfNotExists()) { Console.WriteLine("The \"" + azureBlobContainerName + "\" blob container was not found.\nThe container has now been created."); } else { Console.WriteLine("The \"" + azureBlobContainerName + "\" blob container was found."); } Console.WriteLine(); // Create a reference of Amazon Client AmazonS3Client amazonClient = new AmazonS3Client(amazonAccessKey, amazonSecretKey, Amazon.RegionEndpoint.USEast1); // Initialize dictionary CopyBlobProgress = new Dictionary <string, CopyBlobProgress>(); string[] months = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; DateTime today = DateTime.Now; string todayString = today.ToString("d"); string[] dateParsed = todayString.Split('/'); int day = Int32.Parse(dateParsed[1]); int month; if (Int32.Parse(dateParsed[0]) >= 2) { month = Int32.Parse(dateParsed[0]) - 2; } else { month = Int32.Parse(dateParsed[0]) + 10; } int year; if (day == 1 && Int32.Parse(dateParsed[0]) == 1) { year = Int32.Parse(dateParsed[2]) - 1; } else { year = Int32.Parse(dateParsed[2]); } string continuationToken = null; bool continueListObjects = true; // Since ListObjects returns a maximum of 1000 objects in a single call, // the function will have to be called repeatedly until all objects are returned. while (continueListObjects) { ListObjectsRequest requestToListObjects = new ListObjectsRequest() { BucketName = amazonBucketName, Marker = continuationToken, MaxKeys = 100, }; Console.WriteLine("Attempting to list objects from the \"" + amazonBucketName + "\" S3 AWS bucket..."); // List objects from Amazon S3 var listObjectsResponse = amazonClient.ListObjects(requestToListObjects); // Get the list of objects var objectsFetchedSoFar = listObjectsResponse.S3Objects; Console.WriteLine("Object listing complete. Now beginning the copy process..."); Console.WriteLine(); // See if more objects are available. We'll first process the objects fetched // and continue the loop to get next set of objects. continuationToken = listObjectsResponse.NextMarker; foreach (var s3Object in objectsFetchedSoFar) { string objectKey = s3Object.Key; //if (day == 6 && objectKey.Contains("AWS-") && objectKey.Contains(months[month]) && objectKey.Contains(dateParsed[2])) { } //else if (objectKey.Contains("AWS-") && objectKey.Length == 35) continue; if (day == 1 && !objectKey.Contains("CloudTrail-" + months[month] + "-" + dateParsed[2])) { continue; } else if (day != 1) { if (!objectKey.Contains("CloudTrail-" + months[month + 1] + "-" + dateParsed[2])) { continue; } } // Since ListObjects returns both files and folders, for now we'll skip folders // The way we'll check this is by checking the value of S3 Object Key. If it // end with "/" we'll assume it's a folder. if (objectKey.Contains("CloudTrail-Logs-Final")) { // Construct the URL to copy string urlToCopy = string.Format(CultureInfo.InvariantCulture, objectUrlFormat, amazonBucketName, objectKey); // Create an instance of CloudBlockBlob string azureObjectKey = objectKey.Replace("CloudTrail-Logs-Final/", ""); CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference(azureObjectKey); var blockBlobUrl = blockBlob.Uri.AbsoluteUri; if (!CopyBlobProgress.ContainsKey(blockBlobUrl)) { CopyBlobProgress.Add(blockBlobUrl, new CopyBlobProgress() { Status = CopyStatus.NotStarted, }); // Start the copy process. We would need to put it in try/catch block // as the copy from Amazon S3 will only work with publicly accessible objects. try { Console.WriteLine(string.Format("Attempting to copy from \"{0}\" to \"{1}\"...", urlToCopy, blockBlobUrl)); GetPreSignedUrlRequest request1 = new GetPreSignedUrlRequest() { BucketName = amazonBucketName, Key = objectKey, Expires = DateTime.Now.AddMinutes(1) }; urlToCopy = amazonClient.GetPreSignedURL(request1); blockBlob.StartCopy(new Uri(urlToCopy)); CopyBlobProgress[blockBlobUrl].Status = CopyStatus.Started; } catch (Exception exception) { CopyBlobProgress[blockBlobUrl].Status = CopyStatus.Failed; CopyBlobProgress[blockBlobUrl].Error = exception; } } } } Console.WriteLine(); Console.WriteLine("-"); Console.WriteLine(); Console.WriteLine("Checking the status of copy processes..."); Microsoft.WindowsAzure.Storage.Blob.CopyStatus[] tracker = new Microsoft.WindowsAzure.Storage.Blob.CopyStatus[1000]; // Now track the progress bool checkForBlobCopyStatus = true; while (checkForBlobCopyStatus) { // List blobs in the blob container var blobsList = blobContainer.ListBlobs("", true, BlobListingDetails.Copy); int x = 0; bool written = false; foreach (var blob in blobsList) { var tempBlockBlob = blob as CloudBlob; var copyStatus = tempBlockBlob.CopyState; if (CopyBlobProgress.ContainsKey(tempBlockBlob.Uri.AbsoluteUri)) { var copyBlobProgress = CopyBlobProgress[tempBlockBlob.Uri.AbsoluteUri]; if (copyStatus != null && !copyStatus.Status.Equals(tracker[x])) { tracker[x] = copyStatus.Status; Console.Write(string.Format("Status of \"{0}\" blob copy: ", tempBlockBlob.Uri.AbsoluteUri, copyStatus.Status)); switch (copyStatus.Status) { case Microsoft.WindowsAzure.Storage.Blob.CopyStatus.Aborted: if (copyBlobProgress != null) { copyBlobProgress.Status = CopyStatus.Aborted; Console.WriteLine("Aborted!"); } break; case Microsoft.WindowsAzure.Storage.Blob.CopyStatus.Failed: if (copyBlobProgress != null) { copyBlobProgress.Status = CopyStatus.Failed; Console.WriteLine("Failed!"); } break; case Microsoft.WindowsAzure.Storage.Blob.CopyStatus.Invalid: if (copyBlobProgress != null) { copyBlobProgress.Status = CopyStatus.Invalid; Console.WriteLine("Invalid!"); } break; case Microsoft.WindowsAzure.Storage.Blob.CopyStatus.Pending: if (copyBlobProgress != null) { copyBlobProgress.Status = CopyStatus.Pending; Console.WriteLine("Pending..."); } break; case Microsoft.WindowsAzure.Storage.Blob.CopyStatus.Success: if (copyBlobProgress != null) { copyBlobProgress.Status = CopyStatus.Success; Console.WriteLine("SUCCESS!"); } break; } written = true; } } ++x; } if (written) { Console.WriteLine(); written = false; } var pendingBlob = CopyBlobProgress.FirstOrDefault(c => c.Value.Status == CopyStatus.Pending); if (string.IsNullOrWhiteSpace(pendingBlob.Key)) { checkForBlobCopyStatus = false; } } if (string.IsNullOrWhiteSpace(continuationToken)) { continueListObjects = false; } Console.WriteLine("-"); Console.WriteLine(""); } Console.WriteLine("Process completed!"); Console.Write("Press any key to terminate the program..."); Console.ReadLine(); }
public void GetCopyStateWhenCopyingTest() { StorageBlob.CloudBlobContainer Container = blobUtil.CreateContainer(); string ContainerName = Container.Name; string BlobName = Utility.GenNameString("blockblob"); StorageBlob.CloudBlockBlob Blob = (StorageBlob.CloudBlockBlob)blobUtil.CreateBlockBlob(Container, BlobName); string uri = Test.Data.Get("BigFileUri"); Test.Assert(!String.IsNullOrEmpty(uri), string.Format("Big file uri should be not empty, actually it's {0}", uri)); if (String.IsNullOrEmpty(uri)) { return; } Blob.StartCopy(new Uri(uri)); int maxMonitorTime = 30; //seconds int checkCount = 0; int sleepInterval = 1000; //ms StorageBlob.CopyStatus status = StorageBlob.CopyStatus.Pending; try { int expectedCopyStateCount = 1; do { Test.Info(String.Format("{0}th check current copy state", checkCount)); Test.Assert(agent.GetAzureStorageBlobCopyState(ContainerName, BlobName, false), "Get copy state in dest container should be successed."); Test.Assert(agent.Output.Count == expectedCopyStateCount, String.Format("Should contain {0} copy state, and actually it's {1}", expectedCopyStateCount, agent.Output.Count)); status = (StorageBlob.CopyStatus)agent.Output[0]["Status"]; Test.Assert(status == StorageBlob.CopyStatus.Pending, String.Format("Copy status should be Pending, actually it's {0}", status)); checkCount++; Thread.Sleep(sleepInterval); }while (status == StorageBlob.CopyStatus.Pending && checkCount < maxMonitorTime); Test.Info("Finish the monitor loop and try to abort copy"); try { Blob.AbortCopy(Blob.CopyState.CopyId); } catch (StorageException e) { //TODO use extension method if (e.RequestInformation != null && e.RequestInformation.HttpStatusCode == 409) { Test.Info("Skip 409 abort conflict exception. Error:{0}", e.Message); Test.Info("Detail Error Message: {0}", e.RequestInformation.HttpStatusMessage); } else { Test.AssertFail(String.Format("Can't abort copy. Error: {0}", e.Message)); } } Test.Assert(agent.GetAzureStorageBlobCopyState(ContainerName, BlobName, false), "Get copy state in dest container should be successed."); Test.Assert(agent.Output.Count == expectedCopyStateCount, String.Format("Should contain {0} copy state, and actually it's {1}", expectedCopyStateCount, agent.Output.Count)); status = (StorageBlob.CopyStatus)agent.Output[0]["Status"]; Test.Assert(status == StorageBlob.CopyStatus.Aborted, String.Format("Copy status should be Aborted, actually it's {0}", status)); } finally { blobUtil.RemoveContainer(Container.Name); } }