/// <summary>
/// set azure blob content
/// </summary>
/// <param name="fileName">local file path</param>
/// <param name="containerName">container name</param>
/// <param name="blobName">blob name</param>
/// <returns>null if user cancel the overwrite operation, otherwise return destination blob object</returns>
internal void SetAzureBlobContent(string fileName, string blobName)
{
StorageBlob.BlobType type = StorageBlob.BlobType.BlockBlob;
if (string.Equals(blobType, BlockBlobType, StringComparison.InvariantCultureIgnoreCase))
{
type = StorageBlob.BlobType.BlockBlob;
}
else if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase))
{
type = StorageBlob.BlobType.PageBlob;
}
else if (string.Equals(blobType, AppendBlobType, StringComparison.InvariantCultureIgnoreCase))
{
type = StorageBlob.BlobType.AppendBlob;
}
else
{
throw new InvalidOperationException(string.Format(
CultureInfo.CurrentCulture,
Resources.InvalidBlobType,
blobType,
blobName));
}
if (!string.IsNullOrEmpty(blobName) && !NameUtil.IsValidBlobName(blobName))
{
throw new ArgumentException(String.Format(Resources.InvalidBlobName, blobName));
}
string filePath = GetFullSendFilePath(fileName);
bool isFile = UploadRequests.EnqueueRequest(filePath, type, blobName);
if (!isFile)
{
WriteWarning(String.Format(Resources.CannotSendDirectory, filePath));
}
}
public void SetBlobContentWithMetadata(StorageBlob.BlobType blobType)
{
string filePath = FileUtil.GenerateOneTempTestFile();
StorageBlob.CloudBlobContainer container = blobUtil.CreateContainer();
Hashtable metadata = new Hashtable();
int metaCount = GetRandomTestCount();
for (int i = 0; i < metaCount; i++)
{
string key = Utility.GenRandomAlphabetString();
string value = Utility.GenNameString(string.Empty);
if (!metadata.ContainsKey(key))
{
Test.Info(string.Format("Add meta key: {0} value : {1}", key, value));
metadata.Add(key, value);
}
}
try
{
Test.Assert(agent.SetAzureStorageBlobContent(filePath, container.Name, blobType, string.Empty, true, -1, null, metadata), "set blob content with meta should succeed");
StorageBlob.ICloudBlob blob = container.GetBlobReferenceFromServer(Path.GetFileName(filePath));
blob.FetchAttributes();
ExpectEqual(metadata.Count, blob.Metadata.Count, "meta data count");
foreach (string key in metadata.Keys)
{
ExpectEqual(metadata[key].ToString(), blob.Metadata[key], "Meta data key " + key);
}
}
finally
{
blobUtil.RemoveContainer(container.Name);
FileUtil.RemoveFile(filePath);
}
}
public void ShowBlobWithSpecialChars(BlobType blobType)
{
CloudBlobContainer container = blobUtil.CreateContainer();
string blobName = SpecialChars;
CloudBlob blob = blobUtil.CreateBlob(container, blobName, blobType);
NodeJSAgent nodejsAgent = (NodeJSAgent)CommandAgent;
try
{
Test.Assert(nodejsAgent.ShowAzureStorageBlob(blobName, container.Name), "show blob name with special chars should succeed");
blob.FetchAttributes();
nodejsAgent.OutputValidation(new List <CloudBlob>()
{
blob
});
}
finally
{
blobUtil.RemoveContainer(container.Name);
}
}
public void BlobCmldetsTest(BlobType blobType)
{
string count = Test.Data.Get("BlobCount");
// create temporary source & destination containers
string srcContainer = Utility.GenNameString("src-");
string destContainer = Utility.GenNameString("dest-");
srcBlobHelper.CreateContainer(srcContainer);
destBlobHelper.CreateContainer(destContainer);
int[] counts = Utility.ParseCount(count);
foreach (int fileNum in counts)
{
// prepare blob data
GenerateTestFiles(fileNum);
UploadBlobs(@".\" + FolderName + "-" + fileNum, srcContainer, fileNum, blobType);
foreach (string operation in operations)
{
RunBlobCmdlet(operation, srcContainer, destContainer, fileNum, blobType);
}
// clean generate files
Helper.DeletePattern(FolderName + "-*");
srcBlobHelper.CleanupContainer(srcContainer);
destBlobHelper.CleanupContainer(destContainer);
}
foreach (string operation in operations)
{
WriteResults(operation, fileNumTimes[operation], fileNumTimeSDs[operation]);
}
srcBlobHelper.DeleteContainer(srcContainer);
destBlobHelper.DeleteContainer(destContainer);
}
public static ICloudBlob GetBlob(CloudBlobContainer container, string blobName, StorageType blobType)
{
ICloudBlob blob = null;
if (blobType == StorageType.BlockBlob)
{
blob = container.GetBlockBlobReference(blobName);
}
else
{
blob = container.GetPageBlobReference(blobName);
}
return blob;
}
/// <summary>
/// Functional Cases:
/// 1. Upload a new blob file in the root container (Set-AzureStorageBlobContent Positive 2)
/// 2. Get an existing blob in the root container (Get-AzureStorageBlob Positive 2)
/// 3. Download an existing blob in the root container (Get-AzureStorageBlobContent Positive 2)
/// 4. Remove an existing blob in the root container (Remove-AzureStorageBlob Positive 2)
/// </summary>
internal void RootBlobOperations(Agent agent, string UploadFilePath, string DownloadDirPath, Microsoft.WindowsAzure.Storage.Blob.BlobType Type)
{
const string ROOT_CONTAINER_NAME = "$root";
string blobName = Path.GetFileName(UploadFilePath);
string downloadFilePath = Path.Combine(DownloadDirPath, blobName);
Collection<Dictionary<string, object>> comp = new Collection<Dictionary<string, object>>();
Dictionary<string, object> dic = Utility.GenComparisonData(StorageObjectType.Blob, blobName);
dic["BlobType"] = Type;
comp.Add(dic);
// create the container
CloudBlobContainer container = StorageAccount.CreateCloudBlobClient().GetRootContainerReference();
container.CreateIfNotExists();
//--------------Upload operation--------------
Test.Assert(agent.SetAzureStorageBlobContent(UploadFilePath, ROOT_CONTAINER_NAME, Type), Utility.GenComparisonData("SendAzureStorageBlob", true));
ICloudBlob blob = BlobHelper.QueryBlob(ROOT_CONTAINER_NAME, blobName);
blob.FetchAttributes();
// Verification for returned values
CloudBlobUtil.PackBlobCompareData(blob, dic);
agent.OutputValidation(comp);
Test.Assert(blob.Exists(), "blob " + blobName + " should exist!");
// validate the ContentType value for GetAzureStorageBlob operation
if (Type == Microsoft.WindowsAzure.Storage.Blob.BlobType.BlockBlob)
{
dic["ContentType"] = "application/octet-stream";
}
//--------------Get operation--------------
Test.Assert(agent.GetAzureStorageBlob(blobName, ROOT_CONTAINER_NAME), Utility.GenComparisonData("GetAzureStorageBlob", true));
// Verification for returned values
agent.OutputValidation(comp);
//--------------Download operation--------------
downloadFilePath = Path.Combine(DownloadDirPath, blobName);
Test.Assert(agent.GetAzureStorageBlobContent(blobName, downloadFilePath, ROOT_CONTAINER_NAME),
Utility.GenComparisonData("GetAzureStorageBlobContent", true));
// Verification for returned values
agent.OutputValidation(comp);
Test.Assert(Helper.CompareTwoFiles(downloadFilePath, UploadFilePath),
String.Format("File '{0}' should be bit-wise identicial to '{1}'", downloadFilePath, UploadFilePath));
//--------------Remove operation--------------
Test.Assert(agent.RemoveAzureStorageBlob(blobName, ROOT_CONTAINER_NAME), Utility.GenComparisonData("RemoveAzureStorageBlob", true));
blob = BlobHelper.QueryBlob(ROOT_CONTAINER_NAME, blobName);
Test.Assert(blob == null, "blob {0} should not exist!", blobName);
}
/// <summary>
/// download specified blob
/// </summary>
/// <param name="container"></param>
/// <param name="blob"></param>
private void DownloadBlobFromContainer(StorageBlob.CloudBlobContainer container, StorageBlob.BlobType type)
{
string blobName = Utility.GenNameString("blob");
StorageBlob.ICloudBlob blob = blobUtil.CreateBlob(container, blobName, type);
string filePath = Path.Combine(downloadDirRoot, blob.Name);
Test.Assert(agent.GetAzureStorageBlobContent(blob.Name, filePath, container.Name, true), "download blob should be successful");
string localMd5 = Helper.GetFileContentMD5(filePath);
Test.Assert(localMd5 == blob.Properties.ContentMD5, string.Format("local content md5 should be {0}, and actualy it's {1}", blob.Properties.ContentMD5, localMd5));
agent.OutputValidation(new List <StorageBlob.ICloudBlob> {
blob
});
}
public static CloudBlob GetBlob(CloudBlobContainer container, string blobName, StorageBlobType blobType)
{
switch (blobType)
{
case StorageBlobType.BlockBlob:
return(container.GetBlockBlobReference(blobName));
case StorageBlobType.PageBlob:
return(container.GetPageBlobReference(blobName));
case StorageBlobType.AppendBlob:
return(container.GetAppendBlobReference(blobName));
default:
throw new InvalidOperationException(string.Format("Invalid blob type: {0}", blobType));
}
}
protected Microsoft.WindowsAzure.Storage.Blob.ICloudBlob AssertBlobContainsData(string containerName, string blobName, BlobType blobType, byte[] expectedData)
{
var client = StorageAccount.CreateCloudBlobClient();
var container = client.GetContainerReference(containerName);
if (!container.Exists())
Assert.Fail("AssertBlobContainsData: The container '{0}' does not exist", containerName);
var blob = (blobType == BlobType.BlockBlob
? (ICloudBlob)container.GetBlockBlobReference(blobName)
: (ICloudBlob)container.GetPageBlobReference(blobName));
if (!blob.Exists())
Assert.Fail("AssertBlobContainsData: The blob '{0}' does not exist", blobName);
using (var stream = new MemoryStream())
{
blob.DownloadToStream(stream);
var gottenData = stream.ToArray();
// Comparing strings -> MUCH faster than comparing the raw arrays
var gottenDataString = Convert.ToBase64String(gottenData);
var expectedDataString = Convert.ToBase64String(expectedData);
Assert.AreEqual(expectedData.Length, gottenData.Length);
Assert.AreEqual(gottenDataString, expectedDataString);
}
return blob;
}
/// <summary>
/// Parameters:
/// Block:
/// True for BlockBlob, false for PageBlob
/// </summary>
internal void DownloadBlobTest(Agent agent, string UploadFilePath, string DownloadDirPath, Microsoft.WindowsAzure.Storage.Blob.BlobType Type)
{
string NEW_CONTAINER_NAME = Utility.GenNameString("upload-");
string blobName = Path.GetFileName(UploadFilePath);
Collection <Dictionary <string, object> > comp = new Collection <Dictionary <string, object> >();
Dictionary <string, object> dic = Utility.GenComparisonData(StorageObjectType.Blob, blobName);
dic["BlobType"] = Type;
comp.Add(dic);
// create the container
CloudBlobContainer container = CommonStorageAccount.CreateCloudBlobClient().GetContainerReference(NEW_CONTAINER_NAME);
container.CreateIfNotExists();
try
{
bool bSuccess = false;
// upload the blob file
if (Type == Microsoft.WindowsAzure.Storage.Blob.BlobType.BlockBlob)
{
bSuccess = CommonBlobHelper.UploadFileToBlockBlob(NEW_CONTAINER_NAME, blobName, UploadFilePath);
}
else if (Type == Microsoft.WindowsAzure.Storage.Blob.BlobType.PageBlob)
{
bSuccess = CommonBlobHelper.UploadFileToPageBlob(NEW_CONTAINER_NAME, blobName, UploadFilePath);
}
Test.Assert(bSuccess, "upload file {0} to container {1} should succeed", UploadFilePath, NEW_CONTAINER_NAME);
//--------------Download operation--------------
string downloadFilePath = Path.Combine(DownloadDirPath, blobName);
Test.Assert(agent.GetAzureStorageBlobContent(blobName, downloadFilePath, NEW_CONTAINER_NAME),
Utility.GenComparisonData("GetAzureStorageBlobContent", true));
ICloudBlob blob = CommonBlobHelper.QueryBlob(NEW_CONTAINER_NAME, blobName);
CloudBlobUtil.PackBlobCompareData(blob, dic);
// Verification for returned values
agent.OutputValidation(comp);
Test.Assert(Helper.CompareTwoFiles(downloadFilePath, UploadFilePath),
String.Format("File '{0}' should be bit-wise identicial to '{1}'", downloadFilePath, UploadFilePath));
}
finally
{
// cleanup
container.DeleteIfExists();
}
}
/// <summary>
/// Create a random container with a random blob
/// </summary>
/// <param name="blobNamePrefix">prefix of the blob name</param>
public void SetupTestContainerAndBlob(StorageBlobType type = StorageBlobType.Unspecified)
{
SetupTestContainerAndBlob(TestBase.SpecialChars, type);
}
protected Microsoft.WindowsAzure.Storage.Blob.ICloudBlob AssertBlobContainsData(string containerName, string blobName, BlobType blobType, byte[] expectedData)
{
var client = StorageAccount.CreateCloudBlobClient();
var container = client.GetContainerReference(containerName);
if (!container.Exists())
{
Assert.Fail("AssertBlobContainsData: The container '{0}' does not exist", containerName);
}
var blob = (blobType == BlobType.BlockBlob
? (ICloudBlob)container.GetBlockBlobReference(blobName)
: (ICloudBlob)container.GetPageBlobReference(blobName));
if (!blob.Exists())
{
Assert.Fail("AssertBlobContainsData: The blob '{0}' does not exist", blobName);
}
using (var stream = new MemoryStream())
{
blob.DownloadToStream(stream);
var gottenData = stream.ToArray();
// Comparing strings -> MUCH faster than comparing the raw arrays
var gottenDataString = Convert.ToBase64String(gottenData);
var expectedDataString = Convert.ToBase64String(expectedData);
Assert.AreEqual(expectedData.Length, gottenData.Length);
Assert.AreEqual(gottenDataString, expectedDataString);
}
return(blob);
}
protected Microsoft.WindowsAzure.Storage.Blob.ICloudBlob AssertBlobDoesNotExist(string containerName, string blobName, BlobType blobType)
{
var client = StorageAccount.CreateCloudBlobClient();
var container = client.GetContainerReference(containerName);
if (!container.Exists())
{
Assert.Fail("AssertBlobDoesNotExist: The container '{0}' does not exist", containerName);
}
var blob = (blobType == BlobType.BlockBlob
? (ICloudBlob)container.GetBlockBlobReference(blobName)
: (ICloudBlob)container.GetPageBlobReference(blobName));
if (blob.Exists())
{
Assert.Fail("AssertBlobDoesNotExist: The blob '{0}' exists", blobName);
}
return(blob);
}
internal static void RunBlobCmdlet(string operation, string srcContainer, string destContainer, int fileNum, BlobType blobType)
{
List <long> fileTimeList = new List <long>();
Stopwatch sw = new Stopwatch();
for (int j = 0; j < 5; j++)
{
destBlobHelper.CleanupContainer(destContainer);
PowerShellAgent agent = new PowerShellAgent();
sw.Reset();
switch (operation)
{
case GetBlobs:
sw.Start();
Test.Assert(agent.GetAzureStorageBlob("*", srcContainer), "get blob list should succeed");
break;
case StartCopyBlobs:
agent.AddPipelineScript(string.Format("Get-AzureStorageBlob -Container {0}", srcContainer));
sw.Start();
Test.Assert(agent.StartAzureStorageBlobCopy(string.Empty, string.Empty, destContainer, string.Empty, destContext: destStorageContext), "start copy blob should be successful");
break;
case GetCopyBlobState:
PowerShellAgent.SetStorageContext(destConnectionString);
agent.AddPipelineScript(string.Format("Get-AzureStorageBlob -Container {0}", destContainer));
sw.Start();
Test.Assert(agent.GetAzureStorageBlobCopyState(string.Empty, string.Empty, false), "Get copy state should be success");
break;
case StopCopyBlobs:
PowerShellAgent.SetStorageContext(destConnectionString);
agent.AddPipelineScript(String.Format("Get-AzureStorageBlob -Container {0}", destContainer));
sw.Start();
Test.Assert(agent.StopAzureStorageBlobCopy(string.Empty, string.Empty, "*", true), "Stop multiple copy operations using blob pipeline should be successful");
break;
case RemoveBlobs:
agent.AddPipelineScript(string.Format("Get-AzureStorageBlob -Container {0}", srcContainer));
sw.Start();
Test.Assert(agent.RemoveAzureStorageBlob(string.Empty, string.Empty), "remove blob list should succeed");
break;
default:
throw new Exception("unknown operation : " + operation);
}
sw.Stop();
fileTimeList.Add(sw.ElapsedMilliseconds);
// Verification for returned values
switch (operation)
{
case GetBlobs:
Test.Assert(agent.Output.Count == fileNum, "{0} row returned : {1}", fileNum, agent.Output.Count);
// compare the blob entities
List <CloudBlob> blobList = new List <CloudBlob>();
srcBlobHelper.ListBlobs(srcContainer, out blobList);
agent.OutputValidation(blobList);
break;
case StartCopyBlobs:
Test.Assert(agent.Output.Count == fileNum, "{0} row returned : {1}", fileNum, agent.Output.Count);
break;
}
// set srcConnectionString as default ConnectionString
PowerShellAgent.SetStorageContext(srcConnectionString);
Test.Info("file number : {0} round : {1} {2} time(ms) : {3}", fileNum, j + 1, operation, sw.ElapsedMilliseconds);
}
double average = fileTimeList.Average();
var deviation = fileTimeList.Select(num => Math.Pow(num - average, 2));
double sd = Math.Sqrt(deviation.Average());
fileNumTimes[operation].Add(fileNum, average);
fileNumTimeSDs[operation].Add(fileNum, sd);
Test.Info("file number : {0} average time : {1}", fileNum, average);
Test.Info("file number : {0} standard dev : {1}", fileNum, sd);
}
internal static void UploadBlobs(string folderPath, string containerName, int fileNum, BlobType blobType)
{
Stopwatch sw = new Stopwatch();
PowerShellAgent agent = new PowerShellAgent();
agent.AddPipelineScript(string.Format("ls -File -Path {0}", folderPath));
sw.Start();
Test.Assert(agent.SetAzureStorageBlobContent(string.Empty, containerName, blobType), "upload multiple files should succeed");
sw.Stop();
Test.Info("file number : {0}, upload time(ms) : {1}", fileNum, sw.ElapsedMilliseconds);
}
/// <summary>
/// Create a blob with specified blob type
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">Blob name</param>
/// <param name="type">Blob type</param>
/// <returns>ICloudBlob object</returns>
public StorageBlob.ICloudBlob CreateBlob(StorageBlob.CloudBlobContainer container, string blobName, StorageBlob.BlobType type)
{
if (type == StorageBlob.BlobType.BlockBlob)
{
return(CreateBlockBlob(container, blobName));
}
else
{
return(CreatePageBlob(container, blobName));
}
}
public static StorageBlob.ICloudBlob GetBlob(StorageBlob.CloudBlobContainer container, string blobName, StorageType blobType)
{
StorageBlob.ICloudBlob blob = null;
if (blobType == StorageType.BlockBlob)
{
blob = container.GetBlockBlobReference(blobName);
}
else
{
blob = container.GetPageBlobReference(blobName);
}
return(blob);
}
/// <summary>
/// Create a list of blobs with random properties/metadata/blob type
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">Blob name</param>
/// <param name="BlobType">type</param>
/// <returns>CloudBlob object</returns>
public CloudBlob CreateRandomBlob(CloudBlobContainer container, string blobName, StorageBlobType type = StorageBlobType.Unspecified, bool createBigBlob = false)
{
if (string.IsNullOrEmpty(blobName))
{
blobName = Utility.GenNameString(TestBase.SpecialChars);
}
if (type == StorageBlobType.Unspecified)
{
int randomValue = random.Next(1, 4);
switch (randomValue)
{
case 1:
type = StorageBlobType.PageBlob;
break;
case 2:
type = StorageBlobType.BlockBlob;
break;
case 3:
type = StorageBlobType.AppendBlob;
break;
default:
break;
}
}
if (type == StorageBlobType.PageBlob)
{
return(CreatePageBlob(container, blobName, createBigBlob));
}
else if (type == StorageBlobType.BlockBlob)
{
return(CreateBlockBlob(container, blobName, createBigBlob));
}
else if (type == StorageBlobType.AppendBlob)
{
return(CreateAppendBlob(container, blobName, createBigBlob));
}
else
{
throw new InvalidOperationException(string.Format("Invalid blob type: {0}", type));
}
}
public CloudBlob GetBlobReference(CloudBlobContainer container, string blobName, StorageBlobType type, DateTimeOffset?snapshotTime = null)
{
switch (type)
{
case StorageBlobType.PageBlob:
return(container.GetPageBlobReference(blobName, snapshotTime));
case StorageBlobType.BlockBlob:
return(container.GetBlockBlobReference(blobName, snapshotTime));
case StorageBlobType.AppendBlob:
return(container.GetAppendBlobReference(blobName, snapshotTime));
default:
throw new InvalidOperationException(string.Format("Invalid blob type: {0}", type));
}
}
protected Microsoft.WindowsAzure.Storage.Blob.ICloudBlob AssertBlobDoesNotExist(string containerName, string blobName, BlobType blobType)
{
var client = StorageAccount.CreateCloudBlobClient();
var container = client.GetContainerReference(containerName);
if (!container.Exists())
Assert.Fail("AssertBlobDoesNotExist: The container '{0}' does not exist", containerName);
var blob = (blobType == BlobType.BlockBlob
? (ICloudBlob)container.GetBlockBlobReference(blobName)
: (ICloudBlob)container.GetPageBlobReference(blobName));
if (blob.Exists())
Assert.Fail("AssertBlobDoesNotExist: The blob '{0}' exists", blobName);
return blob;
}
