/// <summary> /// set azure blob content /// </summary> /// <param name="fileName">local file path</param> /// <param name="containerName">container name</param> /// <param name="blobName">blob name</param> /// <returns>null if user cancel the overwrite operation, otherwise return destination blob object</returns> internal void SetAzureBlobContent(string fileName, string blobName) { StorageBlob.BlobType type = StorageBlob.BlobType.BlockBlob; if (string.Equals(blobType, BlockBlobType, StringComparison.InvariantCultureIgnoreCase)) { type = StorageBlob.BlobType.BlockBlob; } else if (string.Equals(blobType, PageBlobType, StringComparison.InvariantCultureIgnoreCase)) { type = StorageBlob.BlobType.PageBlob; } else if (string.Equals(blobType, AppendBlobType, StringComparison.InvariantCultureIgnoreCase)) { type = StorageBlob.BlobType.AppendBlob; } else { throw new InvalidOperationException(string.Format( CultureInfo.CurrentCulture, Resources.InvalidBlobType, blobType, blobName)); } if (!string.IsNullOrEmpty(blobName) && !NameUtil.IsValidBlobName(blobName)) { throw new ArgumentException(String.Format(Resources.InvalidBlobName, blobName)); } string filePath = GetFullSendFilePath(fileName); bool isFile = UploadRequests.EnqueueRequest(filePath, type, blobName); if (!isFile) { WriteWarning(String.Format(Resources.CannotSendDirectory, filePath)); } }
public void SetBlobContentWithMetadata(StorageBlob.BlobType blobType) { string filePath = FileUtil.GenerateOneTempTestFile(); StorageBlob.CloudBlobContainer container = blobUtil.CreateContainer(); Hashtable metadata = new Hashtable(); int metaCount = GetRandomTestCount(); for (int i = 0; i < metaCount; i++) { string key = Utility.GenRandomAlphabetString(); string value = Utility.GenNameString(string.Empty); if (!metadata.ContainsKey(key)) { Test.Info(string.Format("Add meta key: {0} value : {1}", key, value)); metadata.Add(key, value); } } try { Test.Assert(agent.SetAzureStorageBlobContent(filePath, container.Name, blobType, string.Empty, true, -1, null, metadata), "set blob content with meta should succeed"); StorageBlob.ICloudBlob blob = container.GetBlobReferenceFromServer(Path.GetFileName(filePath)); blob.FetchAttributes(); ExpectEqual(metadata.Count, blob.Metadata.Count, "meta data count"); foreach (string key in metadata.Keys) { ExpectEqual(metadata[key].ToString(), blob.Metadata[key], "Meta data key " + key); } } finally { blobUtil.RemoveContainer(container.Name); FileUtil.RemoveFile(filePath); } }
public void ShowBlobWithSpecialChars(BlobType blobType) { CloudBlobContainer container = blobUtil.CreateContainer(); string blobName = SpecialChars; CloudBlob blob = blobUtil.CreateBlob(container, blobName, blobType); NodeJSAgent nodejsAgent = (NodeJSAgent)CommandAgent; try { Test.Assert(nodejsAgent.ShowAzureStorageBlob(blobName, container.Name), "show blob name with special chars should succeed"); blob.FetchAttributes(); nodejsAgent.OutputValidation(new List <CloudBlob>() { blob }); } finally { blobUtil.RemoveContainer(container.Name); } }
public void BlobCmldetsTest(BlobType blobType) { string count = Test.Data.Get("BlobCount"); // create temporary source & destination containers string srcContainer = Utility.GenNameString("src-"); string destContainer = Utility.GenNameString("dest-"); srcBlobHelper.CreateContainer(srcContainer); destBlobHelper.CreateContainer(destContainer); int[] counts = Utility.ParseCount(count); foreach (int fileNum in counts) { // prepare blob data GenerateTestFiles(fileNum); UploadBlobs(@".\" + FolderName + "-" + fileNum, srcContainer, fileNum, blobType); foreach (string operation in operations) { RunBlobCmdlet(operation, srcContainer, destContainer, fileNum, blobType); } // clean generate files Helper.DeletePattern(FolderName + "-*"); srcBlobHelper.CleanupContainer(srcContainer); destBlobHelper.CleanupContainer(destContainer); } foreach (string operation in operations) { WriteResults(operation, fileNumTimes[operation], fileNumTimeSDs[operation]); } srcBlobHelper.DeleteContainer(srcContainer); destBlobHelper.DeleteContainer(destContainer); }
public static ICloudBlob GetBlob(CloudBlobContainer container, string blobName, StorageType blobType) { ICloudBlob blob = null; if (blobType == StorageType.BlockBlob) { blob = container.GetBlockBlobReference(blobName); } else { blob = container.GetPageBlobReference(blobName); } return blob; }
/// <summary> /// Functional Cases: /// 1. Upload a new blob file in the root container (Set-AzureStorageBlobContent Positive 2) /// 2. Get an existing blob in the root container (Get-AzureStorageBlob Positive 2) /// 3. Download an existing blob in the root container (Get-AzureStorageBlobContent Positive 2) /// 4. Remove an existing blob in the root container (Remove-AzureStorageBlob Positive 2) /// </summary> internal void RootBlobOperations(Agent agent, string UploadFilePath, string DownloadDirPath, Microsoft.WindowsAzure.Storage.Blob.BlobType Type) { const string ROOT_CONTAINER_NAME = "$root"; string blobName = Path.GetFileName(UploadFilePath); string downloadFilePath = Path.Combine(DownloadDirPath, blobName); Collection<Dictionary<string, object>> comp = new Collection<Dictionary<string, object>>(); Dictionary<string, object> dic = Utility.GenComparisonData(StorageObjectType.Blob, blobName); dic["BlobType"] = Type; comp.Add(dic); // create the container CloudBlobContainer container = StorageAccount.CreateCloudBlobClient().GetRootContainerReference(); container.CreateIfNotExists(); //--------------Upload operation-------------- Test.Assert(agent.SetAzureStorageBlobContent(UploadFilePath, ROOT_CONTAINER_NAME, Type), Utility.GenComparisonData("SendAzureStorageBlob", true)); ICloudBlob blob = BlobHelper.QueryBlob(ROOT_CONTAINER_NAME, blobName); blob.FetchAttributes(); // Verification for returned values CloudBlobUtil.PackBlobCompareData(blob, dic); agent.OutputValidation(comp); Test.Assert(blob.Exists(), "blob " + blobName + " should exist!"); // validate the ContentType value for GetAzureStorageBlob operation if (Type == Microsoft.WindowsAzure.Storage.Blob.BlobType.BlockBlob) { dic["ContentType"] = "application/octet-stream"; } //--------------Get operation-------------- Test.Assert(agent.GetAzureStorageBlob(blobName, ROOT_CONTAINER_NAME), Utility.GenComparisonData("GetAzureStorageBlob", true)); // Verification for returned values agent.OutputValidation(comp); //--------------Download operation-------------- downloadFilePath = Path.Combine(DownloadDirPath, blobName); Test.Assert(agent.GetAzureStorageBlobContent(blobName, downloadFilePath, ROOT_CONTAINER_NAME), Utility.GenComparisonData("GetAzureStorageBlobContent", true)); // Verification for returned values agent.OutputValidation(comp); Test.Assert(Helper.CompareTwoFiles(downloadFilePath, UploadFilePath), String.Format("File '{0}' should be bit-wise identicial to '{1}'", downloadFilePath, UploadFilePath)); //--------------Remove operation-------------- Test.Assert(agent.RemoveAzureStorageBlob(blobName, ROOT_CONTAINER_NAME), Utility.GenComparisonData("RemoveAzureStorageBlob", true)); blob = BlobHelper.QueryBlob(ROOT_CONTAINER_NAME, blobName); Test.Assert(blob == null, "blob {0} should not exist!", blobName); }
/// <summary> /// download specified blob /// </summary> /// <param name="container"></param> /// <param name="blob"></param> private void DownloadBlobFromContainer(StorageBlob.CloudBlobContainer container, StorageBlob.BlobType type) { string blobName = Utility.GenNameString("blob"); StorageBlob.ICloudBlob blob = blobUtil.CreateBlob(container, blobName, type); string filePath = Path.Combine(downloadDirRoot, blob.Name); Test.Assert(agent.GetAzureStorageBlobContent(blob.Name, filePath, container.Name, true), "download blob should be successful"); string localMd5 = Helper.GetFileContentMD5(filePath); Test.Assert(localMd5 == blob.Properties.ContentMD5, string.Format("local content md5 should be {0}, and actualy it's {1}", blob.Properties.ContentMD5, localMd5)); agent.OutputValidation(new List <StorageBlob.ICloudBlob> { blob }); }
public static CloudBlob GetBlob(CloudBlobContainer container, string blobName, StorageBlobType blobType) { switch (blobType) { case StorageBlobType.BlockBlob: return(container.GetBlockBlobReference(blobName)); case StorageBlobType.PageBlob: return(container.GetPageBlobReference(blobName)); case StorageBlobType.AppendBlob: return(container.GetAppendBlobReference(blobName)); default: throw new InvalidOperationException(string.Format("Invalid blob type: {0}", blobType)); } }
protected Microsoft.WindowsAzure.Storage.Blob.ICloudBlob AssertBlobContainsData(string containerName, string blobName, BlobType blobType, byte[] expectedData) { var client = StorageAccount.CreateCloudBlobClient(); var container = client.GetContainerReference(containerName); if (!container.Exists()) Assert.Fail("AssertBlobContainsData: The container '{0}' does not exist", containerName); var blob = (blobType == BlobType.BlockBlob ? (ICloudBlob)container.GetBlockBlobReference(blobName) : (ICloudBlob)container.GetPageBlobReference(blobName)); if (!blob.Exists()) Assert.Fail("AssertBlobContainsData: The blob '{0}' does not exist", blobName); using (var stream = new MemoryStream()) { blob.DownloadToStream(stream); var gottenData = stream.ToArray(); // Comparing strings -> MUCH faster than comparing the raw arrays var gottenDataString = Convert.ToBase64String(gottenData); var expectedDataString = Convert.ToBase64String(expectedData); Assert.AreEqual(expectedData.Length, gottenData.Length); Assert.AreEqual(gottenDataString, expectedDataString); } return blob; }
/// <summary> /// Parameters: /// Block: /// True for BlockBlob, false for PageBlob /// </summary> internal void DownloadBlobTest(Agent agent, string UploadFilePath, string DownloadDirPath, Microsoft.WindowsAzure.Storage.Blob.BlobType Type) { string NEW_CONTAINER_NAME = Utility.GenNameString("upload-"); string blobName = Path.GetFileName(UploadFilePath); Collection <Dictionary <string, object> > comp = new Collection <Dictionary <string, object> >(); Dictionary <string, object> dic = Utility.GenComparisonData(StorageObjectType.Blob, blobName); dic["BlobType"] = Type; comp.Add(dic); // create the container CloudBlobContainer container = CommonStorageAccount.CreateCloudBlobClient().GetContainerReference(NEW_CONTAINER_NAME); container.CreateIfNotExists(); try { bool bSuccess = false; // upload the blob file if (Type == Microsoft.WindowsAzure.Storage.Blob.BlobType.BlockBlob) { bSuccess = CommonBlobHelper.UploadFileToBlockBlob(NEW_CONTAINER_NAME, blobName, UploadFilePath); } else if (Type == Microsoft.WindowsAzure.Storage.Blob.BlobType.PageBlob) { bSuccess = CommonBlobHelper.UploadFileToPageBlob(NEW_CONTAINER_NAME, blobName, UploadFilePath); } Test.Assert(bSuccess, "upload file {0} to container {1} should succeed", UploadFilePath, NEW_CONTAINER_NAME); //--------------Download operation-------------- string downloadFilePath = Path.Combine(DownloadDirPath, blobName); Test.Assert(agent.GetAzureStorageBlobContent(blobName, downloadFilePath, NEW_CONTAINER_NAME), Utility.GenComparisonData("GetAzureStorageBlobContent", true)); ICloudBlob blob = CommonBlobHelper.QueryBlob(NEW_CONTAINER_NAME, blobName); CloudBlobUtil.PackBlobCompareData(blob, dic); // Verification for returned values agent.OutputValidation(comp); Test.Assert(Helper.CompareTwoFiles(downloadFilePath, UploadFilePath), String.Format("File '{0}' should be bit-wise identicial to '{1}'", downloadFilePath, UploadFilePath)); } finally { // cleanup container.DeleteIfExists(); } }
/// <summary> /// Create a random container with a random blob /// </summary> /// <param name="blobNamePrefix">prefix of the blob name</param> public void SetupTestContainerAndBlob(StorageBlobType type = StorageBlobType.Unspecified) { SetupTestContainerAndBlob(TestBase.SpecialChars, type); }
protected Microsoft.WindowsAzure.Storage.Blob.ICloudBlob AssertBlobContainsData(string containerName, string blobName, BlobType blobType, byte[] expectedData) { var client = StorageAccount.CreateCloudBlobClient(); var container = client.GetContainerReference(containerName); if (!container.Exists()) { Assert.Fail("AssertBlobContainsData: The container '{0}' does not exist", containerName); } var blob = (blobType == BlobType.BlockBlob ? (ICloudBlob)container.GetBlockBlobReference(blobName) : (ICloudBlob)container.GetPageBlobReference(blobName)); if (!blob.Exists()) { Assert.Fail("AssertBlobContainsData: The blob '{0}' does not exist", blobName); } using (var stream = new MemoryStream()) { blob.DownloadToStream(stream); var gottenData = stream.ToArray(); // Comparing strings -> MUCH faster than comparing the raw arrays var gottenDataString = Convert.ToBase64String(gottenData); var expectedDataString = Convert.ToBase64String(expectedData); Assert.AreEqual(expectedData.Length, gottenData.Length); Assert.AreEqual(gottenDataString, expectedDataString); } return(blob); }
protected Microsoft.WindowsAzure.Storage.Blob.ICloudBlob AssertBlobDoesNotExist(string containerName, string blobName, BlobType blobType) { var client = StorageAccount.CreateCloudBlobClient(); var container = client.GetContainerReference(containerName); if (!container.Exists()) { Assert.Fail("AssertBlobDoesNotExist: The container '{0}' does not exist", containerName); } var blob = (blobType == BlobType.BlockBlob ? (ICloudBlob)container.GetBlockBlobReference(blobName) : (ICloudBlob)container.GetPageBlobReference(blobName)); if (blob.Exists()) { Assert.Fail("AssertBlobDoesNotExist: The blob '{0}' exists", blobName); } return(blob); }
internal static void RunBlobCmdlet(string operation, string srcContainer, string destContainer, int fileNum, BlobType blobType) { List <long> fileTimeList = new List <long>(); Stopwatch sw = new Stopwatch(); for (int j = 0; j < 5; j++) { destBlobHelper.CleanupContainer(destContainer); PowerShellAgent agent = new PowerShellAgent(); sw.Reset(); switch (operation) { case GetBlobs: sw.Start(); Test.Assert(agent.GetAzureStorageBlob("*", srcContainer), "get blob list should succeed"); break; case StartCopyBlobs: agent.AddPipelineScript(string.Format("Get-AzureStorageBlob -Container {0}", srcContainer)); sw.Start(); Test.Assert(agent.StartAzureStorageBlobCopy(string.Empty, string.Empty, destContainer, string.Empty, destContext: destStorageContext), "start copy blob should be successful"); break; case GetCopyBlobState: PowerShellAgent.SetStorageContext(destConnectionString); agent.AddPipelineScript(string.Format("Get-AzureStorageBlob -Container {0}", destContainer)); sw.Start(); Test.Assert(agent.GetAzureStorageBlobCopyState(string.Empty, string.Empty, false), "Get copy state should be success"); break; case StopCopyBlobs: PowerShellAgent.SetStorageContext(destConnectionString); agent.AddPipelineScript(String.Format("Get-AzureStorageBlob -Container {0}", destContainer)); sw.Start(); Test.Assert(agent.StopAzureStorageBlobCopy(string.Empty, string.Empty, "*", true), "Stop multiple copy operations using blob pipeline should be successful"); break; case RemoveBlobs: agent.AddPipelineScript(string.Format("Get-AzureStorageBlob -Container {0}", srcContainer)); sw.Start(); Test.Assert(agent.RemoveAzureStorageBlob(string.Empty, string.Empty), "remove blob list should succeed"); break; default: throw new Exception("unknown operation : " + operation); } sw.Stop(); fileTimeList.Add(sw.ElapsedMilliseconds); // Verification for returned values switch (operation) { case GetBlobs: Test.Assert(agent.Output.Count == fileNum, "{0} row returned : {1}", fileNum, agent.Output.Count); // compare the blob entities List <CloudBlob> blobList = new List <CloudBlob>(); srcBlobHelper.ListBlobs(srcContainer, out blobList); agent.OutputValidation(blobList); break; case StartCopyBlobs: Test.Assert(agent.Output.Count == fileNum, "{0} row returned : {1}", fileNum, agent.Output.Count); break; } // set srcConnectionString as default ConnectionString PowerShellAgent.SetStorageContext(srcConnectionString); Test.Info("file number : {0} round : {1} {2} time(ms) : {3}", fileNum, j + 1, operation, sw.ElapsedMilliseconds); } double average = fileTimeList.Average(); var deviation = fileTimeList.Select(num => Math.Pow(num - average, 2)); double sd = Math.Sqrt(deviation.Average()); fileNumTimes[operation].Add(fileNum, average); fileNumTimeSDs[operation].Add(fileNum, sd); Test.Info("file number : {0} average time : {1}", fileNum, average); Test.Info("file number : {0} standard dev : {1}", fileNum, sd); }
internal static void UploadBlobs(string folderPath, string containerName, int fileNum, BlobType blobType) { Stopwatch sw = new Stopwatch(); PowerShellAgent agent = new PowerShellAgent(); agent.AddPipelineScript(string.Format("ls -File -Path {0}", folderPath)); sw.Start(); Test.Assert(agent.SetAzureStorageBlobContent(string.Empty, containerName, blobType), "upload multiple files should succeed"); sw.Stop(); Test.Info("file number : {0}, upload time(ms) : {1}", fileNum, sw.ElapsedMilliseconds); }
/// <summary> /// Create a blob with specified blob type /// </summary> /// <param name="container">CloudBlobContainer object</param> /// <param name="blobName">Blob name</param> /// <param name="type">Blob type</param> /// <returns>ICloudBlob object</returns> public StorageBlob.ICloudBlob CreateBlob(StorageBlob.CloudBlobContainer container, string blobName, StorageBlob.BlobType type) { if (type == StorageBlob.BlobType.BlockBlob) { return(CreateBlockBlob(container, blobName)); } else { return(CreatePageBlob(container, blobName)); } }
public static StorageBlob.ICloudBlob GetBlob(StorageBlob.CloudBlobContainer container, string blobName, StorageType blobType) { StorageBlob.ICloudBlob blob = null; if (blobType == StorageType.BlockBlob) { blob = container.GetBlockBlobReference(blobName); } else { blob = container.GetPageBlobReference(blobName); } return(blob); }
/// <summary> /// Create a list of blobs with random properties/metadata/blob type /// </summary> /// <param name="container">CloudBlobContainer object</param> /// <param name="blobName">Blob name</param> /// <param name="BlobType">type</param> /// <returns>CloudBlob object</returns> public CloudBlob CreateRandomBlob(CloudBlobContainer container, string blobName, StorageBlobType type = StorageBlobType.Unspecified, bool createBigBlob = false) { if (string.IsNullOrEmpty(blobName)) { blobName = Utility.GenNameString(TestBase.SpecialChars); } if (type == StorageBlobType.Unspecified) { int randomValue = random.Next(1, 4); switch (randomValue) { case 1: type = StorageBlobType.PageBlob; break; case 2: type = StorageBlobType.BlockBlob; break; case 3: type = StorageBlobType.AppendBlob; break; default: break; } } if (type == StorageBlobType.PageBlob) { return(CreatePageBlob(container, blobName, createBigBlob)); } else if (type == StorageBlobType.BlockBlob) { return(CreateBlockBlob(container, blobName, createBigBlob)); } else if (type == StorageBlobType.AppendBlob) { return(CreateAppendBlob(container, blobName, createBigBlob)); } else { throw new InvalidOperationException(string.Format("Invalid blob type: {0}", type)); } }
public CloudBlob GetBlobReference(CloudBlobContainer container, string blobName, StorageBlobType type, DateTimeOffset?snapshotTime = null) { switch (type) { case StorageBlobType.PageBlob: return(container.GetPageBlobReference(blobName, snapshotTime)); case StorageBlobType.BlockBlob: return(container.GetBlockBlobReference(blobName, snapshotTime)); case StorageBlobType.AppendBlob: return(container.GetAppendBlobReference(blobName, snapshotTime)); default: throw new InvalidOperationException(string.Format("Invalid blob type: {0}", type)); } }
protected Microsoft.WindowsAzure.Storage.Blob.ICloudBlob AssertBlobDoesNotExist(string containerName, string blobName, BlobType blobType) { var client = StorageAccount.CreateCloudBlobClient(); var container = client.GetContainerReference(containerName); if (!container.Exists()) Assert.Fail("AssertBlobDoesNotExist: The container '{0}' does not exist", containerName); var blob = (blobType == BlobType.BlockBlob ? (ICloudBlob)container.GetBlockBlobReference(blobName) : (ICloudBlob)container.GetPageBlobReference(blobName)); if (blob.Exists()) Assert.Fail("AssertBlobDoesNotExist: The blob '{0}' exists", blobName); return blob; }