public void GetBlobCopyStateTest()
{
CloudBlobUtil blobUtil = new CloudBlobUtil(CommonStorageAccount);
blobUtil.SetupTestContainerAndBlob();
StorageBlob.ICloudBlob destBlob = CopyBlobAndWaitForComplete(blobUtil);
try
{
Test.Assert(destBlob.CopyState.Status == StorageBlob.CopyStatus.Success, String.Format("The blob copy using storage client should be success, actually it's {0}", destBlob.CopyState.Status));
PowerShellAgent agent = new PowerShellAgent();
Test.Assert(agent.GetAzureStorageBlobCopyState(blobUtil.ContainerName, destBlob.Name, false), "Get copy state should be success");
int expectedStateCount = 1;
Test.Assert(agent.Output.Count == expectedStateCount, String.Format("Expected to get {0} copy state, actually it's {1}", expectedStateCount, agent.Output.Count));
StorageBlob.CopyStatus copyStatus = (StorageBlob.CopyStatus)agent.Output[0]["Status"];
Test.Assert(copyStatus == StorageBlob.CopyStatus.Success, String.Format("The blob copy should be success, actually it's {0}", copyStatus));
Uri sourceUri = (Uri)agent.Output[0]["Source"];
string expectedUri = CloudBlobUtil.ConvertCopySourceUri(blobUtil.Blob.Uri.ToString());
Test.Assert(sourceUri.ToString() == expectedUri, String.Format("Expected source uri is {0}, actully it's {1}", expectedUri, sourceUri.ToString()));
Test.Assert(!agent.GetAzureStorageBlobCopyState(blobUtil.ContainerName, blobUtil.BlobName, false), "Get copy state should be fail since the specified blob don't have any copy operation");
Test.Assert(agent.ErrorMessages.Count > 0, "Should return error message");
string errorMessage = "Can not find copy task on specified blob";
Test.Assert(agent.ErrorMessages[0].StartsWith(errorMessage), String.Format("Error message should start with {0}, and actually it's {1}", errorMessage, agent.ErrorMessages[0]));
}
finally
{
blobUtil.CleanupTestContainerAndBlob();
}
}
private void AssertFinishedCopyState(Uri SourceUri, int startIndex = 0)
{
string expectedSourceUri = CloudBlobUtil.ConvertCopySourceUri(SourceUri.ToString());
Test.Assert(agent.Output.Count > startIndex, String.Format("Should contain the great than {0} copy state, and actually it's {1}", startIndex, agent.Output.Count));
string sourceUri = ((Uri)agent.Output[startIndex]["Source"]).ToString();
Test.Assert(sourceUri.StartsWith(expectedSourceUri), String.Format("source uri should start with {0}, and actualy it's {1}", expectedSourceUri, sourceUri));
StorageBlob.CopyStatus status = (StorageBlob.CopyStatus)agent.Output[startIndex]["Status"];
Test.Assert(status != StorageBlob.CopyStatus.Pending, String.Format("Copy status should not be Pending, actually it's {0}", status));
string copyId = (string)agent.Output[startIndex]["CopyId"];
Test.Assert(!String.IsNullOrEmpty(copyId), "Copy ID should be not empty");
}
static void Main(string[] args)
{
// Create a reference of Windows Azure Storage Account
StorageCredentials azureCloudStorageCredentials = new StorageCredentials(azureStorageAccountName, azureStorageAccountKey);
CloudStorageAccount azureCloudStorageAccount = new CloudStorageAccount(azureCloudStorageCredentials, true);
// Get a reference of Blob Container where the objects will be copied
var blobContainer = azureCloudStorageAccount.CreateCloudBlobClient().GetContainerReference(azureBlobContainerName);
// Create blob container if needed
Console.WriteLine("Searching for the \"" + azureBlobContainerName + "\" blob container...");
if (blobContainer.CreateIfNotExists())
{
Console.WriteLine("The \"" + azureBlobContainerName + "\" blob container was not found.\nThe container has now been created.");
}
else
{
Console.WriteLine("The \"" + azureBlobContainerName + "\" blob container was found.");
}
Console.WriteLine();
// Create a reference of Amazon Client
AmazonS3Client amazonClient = new AmazonS3Client(amazonAccessKey, amazonSecretKey, Amazon.RegionEndpoint.USEast1);
// Initialize dictionary
CopyBlobProgress = new Dictionary <string, CopyBlobProgress>();
string[] months = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
DateTime today = DateTime.Now;
string todayString = today.ToString("d");
string[] dateParsed = todayString.Split('/');
int day = Int32.Parse(dateParsed[1]);
int month;
if (Int32.Parse(dateParsed[0]) >= 2)
{
month = Int32.Parse(dateParsed[0]) - 2;
}
else
{
month = Int32.Parse(dateParsed[0]) + 10;
}
int year;
if (day == 1 && Int32.Parse(dateParsed[0]) == 1)
{
year = Int32.Parse(dateParsed[2]) - 1;
}
else
{
year = Int32.Parse(dateParsed[2]);
}
string continuationToken = null;
bool continueListObjects = true;
// Since ListObjects returns a maximum of 1000 objects in a single call,
// the function will have to be called repeatedly until all objects are returned.
while (continueListObjects)
{
ListObjectsRequest requestToListObjects = new ListObjectsRequest()
{
BucketName = amazonBucketName,
Marker = continuationToken,
MaxKeys = 100,
};
Console.WriteLine("Attempting to list objects from the \"" + amazonBucketName + "\" S3 AWS bucket...");
// List objects from Amazon S3
var listObjectsResponse = amazonClient.ListObjects(requestToListObjects);
// Get the list of objects
var objectsFetchedSoFar = listObjectsResponse.S3Objects;
Console.WriteLine("Object listing complete. Now beginning the copy process...");
Console.WriteLine();
// See if more objects are available. We'll first process the objects fetched
// and continue the loop to get next set of objects.
continuationToken = listObjectsResponse.NextMarker;
foreach (var s3Object in objectsFetchedSoFar)
{
string objectKey = s3Object.Key;
//if (day == 6 && objectKey.Contains("AWS-") && objectKey.Contains(months[month]) && objectKey.Contains(dateParsed[2])) { }
//else if (objectKey.Contains("AWS-") && objectKey.Length == 35) continue;
if (day == 1 && !objectKey.Contains("CloudTrail-" + months[month] + "-" + dateParsed[2]))
{
continue;
}
else if (day != 1)
{
if (!objectKey.Contains("CloudTrail-" + months[month + 1] + "-" + dateParsed[2]))
{
continue;
}
}
// Since ListObjects returns both files and folders, for now we'll skip folders
// The way we'll check this is by checking the value of S3 Object Key. If it
// end with "/" we'll assume it's a folder.
if (objectKey.Contains("CloudTrail-Logs-Final"))
{
// Construct the URL to copy
string urlToCopy = string.Format(CultureInfo.InvariantCulture, objectUrlFormat, amazonBucketName, objectKey);
// Create an instance of CloudBlockBlob
string azureObjectKey = objectKey.Replace("CloudTrail-Logs-Final/", "");
CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference(azureObjectKey);
var blockBlobUrl = blockBlob.Uri.AbsoluteUri;
if (!CopyBlobProgress.ContainsKey(blockBlobUrl))
{
CopyBlobProgress.Add(blockBlobUrl, new CopyBlobProgress()
{
Status = CopyStatus.NotStarted,
});
// Start the copy process. We would need to put it in try/catch block
// as the copy from Amazon S3 will only work with publicly accessible objects.
try {
Console.WriteLine(string.Format("Attempting to copy from \"{0}\" to \"{1}\"...", urlToCopy, blockBlobUrl));
GetPreSignedUrlRequest request1 = new GetPreSignedUrlRequest()
{
BucketName = amazonBucketName,
Key = objectKey,
Expires = DateTime.Now.AddMinutes(1)
};
urlToCopy = amazonClient.GetPreSignedURL(request1);
blockBlob.StartCopy(new Uri(urlToCopy));
CopyBlobProgress[blockBlobUrl].Status = CopyStatus.Started;
} catch (Exception exception) {
CopyBlobProgress[blockBlobUrl].Status = CopyStatus.Failed;
CopyBlobProgress[blockBlobUrl].Error = exception;
}
}
}
}
Console.WriteLine();
Console.WriteLine("-");
Console.WriteLine();
Console.WriteLine("Checking the status of copy processes...");
Microsoft.WindowsAzure.Storage.Blob.CopyStatus[] tracker = new Microsoft.WindowsAzure.Storage.Blob.CopyStatus[1000];
// Now track the progress
bool checkForBlobCopyStatus = true;
while (checkForBlobCopyStatus)
{
// List blobs in the blob container
var blobsList = blobContainer.ListBlobs("", true, BlobListingDetails.Copy);
int x = 0;
bool written = false;
foreach (var blob in blobsList)
{
var tempBlockBlob = blob as CloudBlob;
var copyStatus = tempBlockBlob.CopyState;
if (CopyBlobProgress.ContainsKey(tempBlockBlob.Uri.AbsoluteUri))
{
var copyBlobProgress = CopyBlobProgress[tempBlockBlob.Uri.AbsoluteUri];
if (copyStatus != null && !copyStatus.Status.Equals(tracker[x]))
{
tracker[x] = copyStatus.Status;
Console.Write(string.Format("Status of \"{0}\" blob copy: ", tempBlockBlob.Uri.AbsoluteUri, copyStatus.Status));
switch (copyStatus.Status)
{
case Microsoft.WindowsAzure.Storage.Blob.CopyStatus.Aborted:
if (copyBlobProgress != null)
{
copyBlobProgress.Status = CopyStatus.Aborted;
Console.WriteLine("Aborted!");
}
break;
case Microsoft.WindowsAzure.Storage.Blob.CopyStatus.Failed:
if (copyBlobProgress != null)
{
copyBlobProgress.Status = CopyStatus.Failed;
Console.WriteLine("Failed!");
}
break;
case Microsoft.WindowsAzure.Storage.Blob.CopyStatus.Invalid:
if (copyBlobProgress != null)
{
copyBlobProgress.Status = CopyStatus.Invalid;
Console.WriteLine("Invalid!");
}
break;
case Microsoft.WindowsAzure.Storage.Blob.CopyStatus.Pending:
if (copyBlobProgress != null)
{
copyBlobProgress.Status = CopyStatus.Pending;
Console.WriteLine("Pending...");
}
break;
case Microsoft.WindowsAzure.Storage.Blob.CopyStatus.Success:
if (copyBlobProgress != null)
{
copyBlobProgress.Status = CopyStatus.Success;
Console.WriteLine("SUCCESS!");
}
break;
}
written = true;
}
}
++x;
}
if (written)
{
Console.WriteLine();
written = false;
}
var pendingBlob = CopyBlobProgress.FirstOrDefault(c => c.Value.Status == CopyStatus.Pending);
if (string.IsNullOrWhiteSpace(pendingBlob.Key))
{
checkForBlobCopyStatus = false;
}
}
if (string.IsNullOrWhiteSpace(continuationToken))
{
continueListObjects = false;
}
Console.WriteLine("-");
Console.WriteLine("");
}
Console.WriteLine("Process completed!");
Console.Write("Press any key to terminate the program...");
Console.ReadLine();
}
public void GetCopyStateWhenCopyingTest()
{
StorageBlob.CloudBlobContainer Container = blobUtil.CreateContainer();
string ContainerName = Container.Name;
string BlobName = Utility.GenNameString("blockblob");
StorageBlob.CloudBlockBlob Blob = (StorageBlob.CloudBlockBlob)blobUtil.CreateBlockBlob(Container, BlobName);
string uri = Test.Data.Get("BigFileUri");
Test.Assert(!String.IsNullOrEmpty(uri), string.Format("Big file uri should be not empty, actually it's {0}", uri));
if (String.IsNullOrEmpty(uri))
{
return;
}
Blob.StartCopy(new Uri(uri));
int maxMonitorTime = 30; //seconds
int checkCount = 0;
int sleepInterval = 1000; //ms
StorageBlob.CopyStatus status = StorageBlob.CopyStatus.Pending;
try
{
int expectedCopyStateCount = 1;
do
{
Test.Info(String.Format("{0}th check current copy state", checkCount));
Test.Assert(agent.GetAzureStorageBlobCopyState(ContainerName, BlobName, false), "Get copy state in dest container should be successed.");
Test.Assert(agent.Output.Count == expectedCopyStateCount, String.Format("Should contain {0} copy state, and actually it's {1}", expectedCopyStateCount, agent.Output.Count));
status = (StorageBlob.CopyStatus)agent.Output[0]["Status"];
Test.Assert(status == StorageBlob.CopyStatus.Pending, String.Format("Copy status should be Pending, actually it's {0}", status));
checkCount++;
Thread.Sleep(sleepInterval);
}while (status == StorageBlob.CopyStatus.Pending && checkCount < maxMonitorTime);
Test.Info("Finish the monitor loop and try to abort copy");
try
{
Blob.AbortCopy(Blob.CopyState.CopyId);
}
catch (StorageException e)
{
//TODO use extension method
if (e.RequestInformation != null && e.RequestInformation.HttpStatusCode == 409)
{
Test.Info("Skip 409 abort conflict exception. Error:{0}", e.Message);
Test.Info("Detail Error Message: {0}", e.RequestInformation.HttpStatusMessage);
}
else
{
Test.AssertFail(String.Format("Can't abort copy. Error: {0}", e.Message));
}
}
Test.Assert(agent.GetAzureStorageBlobCopyState(ContainerName, BlobName, false), "Get copy state in dest container should be successed.");
Test.Assert(agent.Output.Count == expectedCopyStateCount, String.Format("Should contain {0} copy state, and actually it's {1}", expectedCopyStateCount, agent.Output.Count));
status = (StorageBlob.CopyStatus)agent.Output[0]["Status"];
Test.Assert(status == StorageBlob.CopyStatus.Aborted, String.Format("Copy status should be Aborted, actually it's {0}", status));
}
finally
{
blobUtil.RemoveContainer(Container.Name);
}
}
