public async Task UpdateFireblob(CloudBlobContainer blobContainer)
{
try
{
IFirebaseConfig config = new FirebaseConfig
{
AuthSecret = "Gg5t1fSLC0WWPVM1VMoNxlM29qO1s53dEso7Jrfp",
BasePath = "https://ringtoneapp.firebaseio.com/"
};
IFirebaseClient client = new FirebaseClient(config);
var list = blobContainer.ListBlobs();
List<CloudBlockBlob> blobNames = list.OfType<CloudBlockBlob>().ToList();
// SET
var todo = new Todo();
List<Todo> todoList = new List<Todo>();
List<UploadDataModel> MetaList = new List<UploadDataModel>();
foreach (var blb in blobNames)
{
blb.FetchAttributes();
Todo td = new Todo();
td.name = blb.Name;
td.url = blb.Uri.AbsoluteUri.ToString();
if (blb.Metadata.Values.Count > 0)
{
td.Category = blb.Metadata.Values.ElementAt(0);
td.UserName = blb.Metadata.Values.ElementAt(1);
td.Number = blb.Metadata.Values.ElementAt(2);
td.Email = blb.Metadata.Values.ElementAt(3);
td.Comments = blb.Metadata.Values.ElementAt(4);
}
todoList.Add(td);
}
SetResponse response = await client.SetAsync(FirebaseContainer, todoList);
List<Todo> setresult = response.ResultAs<List<Todo>>();
}
catch (Exception e)
{
}
//GET
//FirebaseResponse getresponse = await client.GetAsync("ringtones");
//List<Todo> gettodo = response.ResultAs<List<Todo>>(); //The response will contain the data being retreived
}
public void SetPageBlobWithInvalidFileSize()
{
string fileName = Utility.GenNameString("tinypageblob");
string filePath = Path.Combine(uploadDirRoot, fileName);
int fileSize = 480;
Helper.GenerateTinyFile(filePath, fileSize);
string containerName = Utility.GenNameString("container");
StorageBlob.CloudBlobContainer container = blobUtil.CreateContainer(containerName);
try
{
List <StorageBlob.IListBlobItem> blobLists = container.ListBlobs(string.Empty, true, StorageBlob.BlobListingDetails.All).ToList();
Test.Assert(blobLists.Count == 0, string.Format("container {0} should contain {1} blobs, and actually it contain {2} blobs", containerName, 0, blobLists.Count));
Test.Assert(!agent.SetAzureStorageBlobContent(filePath, containerName, StorageBlob.BlobType.PageBlob), "upload page blob with invalid file size should be failed.");
string expectedErrorMessage = "The page blob size must be a multiple of 512 bytes.";
Test.Assert(agent.ErrorMessages[0].StartsWith(expectedErrorMessage), expectedErrorMessage);
blobLists = container.ListBlobs(string.Empty, true, StorageBlob.BlobListingDetails.All).ToList();
Test.Assert(blobLists.Count == 0, string.Format("container {0} should contain {1} blobs, and actually it contain {2} blobs", containerName, 0, blobLists.Count));
}
finally
{
blobUtil.RemoveContainer(containerName);
FileUtil.RemoveFile(filePath);
}
}
public void SetBlobContentWithInvalidBlobType()
{
string containerName = Utility.GenNameString("container");
StorageBlob.CloudBlobContainer container = blobUtil.CreateContainer(containerName);
try
{
string blobName = files[0];
List <StorageBlob.IListBlobItem> blobLists = container.ListBlobs(string.Empty, true, StorageBlob.BlobListingDetails.All).ToList();
Test.Assert(blobLists.Count == 0, string.Format("container {0} should contain {1} blobs, and actually it contain {2} blobs", containerName, 0, blobLists.Count));
Test.Assert(agent.SetAzureStorageBlobContent(Path.Combine(uploadDirRoot, files[0]), containerName, StorageBlob.BlobType.BlockBlob, blobName), "upload blob should be successful.");
blobLists = container.ListBlobs(string.Empty, true, StorageBlob.BlobListingDetails.All).ToList();
Test.Assert(blobLists.Count == 1, string.Format("container {0} should contain {1} blobs, and actually it contain {2} blobs", containerName, 1, blobLists.Count));
string convertBlobName = blobUtil.ConvertFileNameToBlobName(blobName);
Test.Assert(((StorageBlob.ICloudBlob)blobLists[0]).Name == convertBlobName, string.Format("blob name should be {0}, actually it's {1}", convertBlobName, ((StorageBlob.ICloudBlob)blobLists[0]).Name));
Test.Assert(!agent.SetAzureStorageBlobContent(Path.Combine(uploadDirRoot, files[0]), containerName, StorageBlob.BlobType.PageBlob, blobName), "upload blob should be with invalid blob should be failed.");
string expectedErrorMessage = string.Format("Blob type mismatched, the current blob type of '{0}' is BlockBlob.", ((StorageBlob.ICloudBlob)blobLists[0]).Name);
Test.Assert(agent.ErrorMessages[0] == expectedErrorMessage, string.Format("Expect error message: {0} != {1}", expectedErrorMessage, agent.ErrorMessages[0]));
}
finally
{
blobUtil.RemoveContainer(containerName);
}
}
public static void ListBlobs(CloudBlobContainer blobContainer, BlobListing blobListing, String prefix = null)
{
if (blobListing == BlobListing.Flat)
{
foreach (var blockBlob in blobContainer.ListBlobs(prefix: null, useFlatBlobListing: true))
{
Console.WriteLine("{0,-10} - {1}", @"Blob", ((CloudBlockBlob)blockBlob).Name);
Console.WriteLine("{0,-10} - {1}", String.Empty, blockBlob.Uri);
}
}
else if (blobListing == BlobListing.Hierarchical)
{
foreach (var blockBlob in blobContainer.ListBlobs(prefix: prefix, useFlatBlobListing: false))
{
if (blockBlob is CloudBlockBlob)
{
Console.WriteLine("{0,-10} - {1}", @"Blob", ((CloudBlockBlob)blockBlob).Name);
Console.WriteLine("{0,-10} - {1}", String.Empty, blockBlob.Uri);
}
else if (blockBlob is CloudBlobDirectory)
{
Console.WriteLine("{0,-10} - {1}", @"Directory", ((CloudBlobDirectory)blockBlob).Prefix);
Console.WriteLine("{0,-10} - {1}", String.Empty, blockBlob.Uri);
ListBlobs(blobContainer, BlobListing.Hierarchical, ((CloudBlobDirectory)blockBlob).Prefix);
}
}
}
}
public void SetBlobContentWithSubDirectory()
{
DirectoryInfo rootDir = new DirectoryInfo(uploadDirRoot);
DirectoryInfo[] dirs = rootDir.GetDirectories();
foreach (DirectoryInfo dir in dirs)
{
string containerName = Utility.GenNameString("container");
StorageBlob.CloudBlobContainer container = blobUtil.CreateContainer(containerName);
try
{
List <StorageBlob.IListBlobItem> blobLists = container.ListBlobs(string.Empty, true, StorageBlob.BlobListingDetails.All).ToList();
Test.Assert(blobLists.Count == 0, string.Format("container {0} should contain {1} blobs, and actually it contain {2} blobs", containerName, 0, blobLists.Count));
StorageBlob.BlobType blobType = StorageBlob.BlobType.BlockBlob;
if (dir.Name.StartsWith("dirpage"))
{
blobType = Microsoft.WindowsAzure.Storage.Blob.BlobType.PageBlob;
}
((PowerShellAgent)agent).AddPipelineScript(string.Format("ls -File -Recurse -Path {0}", dir.FullName));
Test.Info("Upload files...");
Test.Assert(agent.SetAzureStorageBlobContent(string.Empty, containerName, blobType), "upload multiple files should be successsed");
Test.Info("Upload finished...");
blobLists = container.ListBlobs(string.Empty, true, StorageBlob.BlobListingDetails.All).ToList();
List <string> dirFiles = files.FindAll(item => item.StartsWith(dir.Name));
Test.Assert(blobLists.Count == dirFiles.Count(), string.Format("set-azurestorageblobcontent should upload {0} files, and actually it's {1}", dirFiles.Count(), blobLists.Count));
StorageBlob.ICloudBlob blob = null;
for (int i = 0, count = dirFiles.Count(); i < count; i++)
{
blob = blobLists[i] as StorageBlob.ICloudBlob;
if (blob == null)
{
Test.AssertFail("blob can't be null");
}
string convertedName = blobUtil.ConvertBlobNameToFileName(blob.Name, dir.Name);
Test.Assert(dirFiles[i] == convertedName, string.Format("blob name should be {0}, and actully it's {1}", dirFiles[i], convertedName));
string localMd5 = Helper.GetFileContentMD5(Path.Combine(uploadDirRoot, dirFiles[i]));
Test.Assert(blob.BlobType == blobType, "blob type should be block blob");
Test.Assert(localMd5 == blob.Properties.ContentMD5, string.Format("blob content md5 should be {0}, and actualy it's {1}", localMd5, blob.Properties.ContentMD5));
}
}
finally
{
blobUtil.RemoveContainer(containerName);
}
}
}
public List<CloudBlockBlob> GetBlobs(CloudBlobContainer container, string filter = "")
{
return container.ListBlobs()
.OfType<CloudBlockBlob>()
.Where(b => String.IsNullOrEmpty(filter) || b.Name.Contains(filter))
.ToList();
}
public BlobFileProvider(IEnumerable<string> locations)
: base()
{
_storageAccount = CloudStorageAccount.Parse(ConfigurationManager.ConnectionStrings["StorageConnectionString"].ConnectionString);
_blobClient = _storageAccount.CreateCloudBlobClient();
_container = _blobClient.GetContainerReference("data");
_container.CreateIfNotExists();
_strings = new List<string>();
foreach(string location in locations) {
foreach (IListBlobItem item in _container.ListBlobs(location,true))
{
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob blob = (CloudBlockBlob)item;
string text;
using (var memoryStream = new MemoryStream())
{
blob.DownloadToStream(memoryStream);
text = Encoding.UTF8.GetString(memoryStream.ToArray());
if (text[0] == _byteOrderMarkUtf8[0])
{
text= text.Remove(0,_byteOrderMarkUtf8.Length);
}
_strings.Add(text);
}
}
}
}
}
public void GetCopyStateFromMultiBlobsTest()
{
StorageBlob.CloudBlobContainer srcContainer = blobUtil.CreateContainer();
StorageBlob.CloudBlobContainer destContainer = blobUtil.CreateContainer();
List <StorageBlob.ICloudBlob> blobs = blobUtil.CreateRandomBlob(srcContainer);
try
{
((PowerShellAgent)agent).AddPipelineScript(String.Format("Get-AzureStorageBlob -Container {0}", srcContainer.Name));
((PowerShellAgent)agent).AddPipelineScript(String.Format("Start-AzureStorageBlobCopy -DestContainer {0}", destContainer.Name));
Test.Assert(agent.GetAzureStorageBlobCopyState(string.Empty, string.Empty, true), "Get copy state for many blobs should be successed.");
Test.Assert(agent.Output.Count == blobs.Count, String.Format("Expected get {0} copy state, and actually get {1} copy state", blobs.Count, agent.Output.Count));
List <StorageBlob.IListBlobItem> destBlobs = destContainer.ListBlobs().ToList();
Test.Assert(destBlobs.Count == blobs.Count, String.Format("Expected get {0} copied blobs, and actually get {1} copy state", blobs.Count, destBlobs.Count));
for (int i = 0, count = agent.Output.Count; i < count; i++)
{
AssertFinishedCopyState(blobs[i].Uri, i);
}
}
finally
{
blobUtil.RemoveContainer(srcContainer.Name);
blobUtil.RemoveContainer(destContainer.Name);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="AzureBlobStorageUploader"/> class
/// </summary>
/// <param name="files">The file collection to upload</param>
/// <param name="container">The Azure Blob Container to upload to</param>
public AzureBlobStorageUploader(UploadableFileCollection files, CloudBlobContainer container)
: base(files)
{
_container = container;
var existingFileNames = new List<string>(container.ListBlobs(null, false)
.Select(n => WebUtility.UrlDecode(n.Uri.Segments.Last()))).ToArray();
files.AssertAndResolveUniqueSaveNames(existingFileNames);
}
public void SetBlobContentByMultipleFiles()
{
string containerName = Utility.GenNameString("container");
StorageBlob.CloudBlobContainer container = blobUtil.CreateContainer(containerName);
try
{
List <StorageBlob.IListBlobItem> blobLists = container.ListBlobs(string.Empty, true, StorageBlob.BlobListingDetails.All).ToList();
Test.Assert(blobLists.Count == 0, string.Format("container {0} should contain {1} blobs, and actually it contain {2} blobs", containerName, 0, blobLists.Count));
DirectoryInfo rootDir = new DirectoryInfo(uploadDirRoot);
FileInfo[] rootFiles = rootDir.GetFiles();
((PowerShellAgent)agent).AddPipelineScript(string.Format("ls -File -Path {0}", uploadDirRoot));
Test.Info("Upload files...");
Test.Assert(agent.SetAzureStorageBlobContent(string.Empty, containerName, StorageBlob.BlobType.BlockBlob), "upload multiple files should be successsed");
Test.Info("Upload finished...");
blobLists = container.ListBlobs(string.Empty, true, StorageBlob.BlobListingDetails.All).ToList();
Test.Assert(blobLists.Count == rootFiles.Count(), string.Format("set-azurestorageblobcontent should upload {0} files, and actually it's {1}", rootFiles.Count(), blobLists.Count));
StorageBlob.ICloudBlob blob = null;
for (int i = 0, count = rootFiles.Count(); i < count; i++)
{
blob = blobLists[i] as StorageBlob.ICloudBlob;
if (blob == null)
{
Test.AssertFail("blob can't be null");
}
Test.Assert(rootFiles[i].Name == blob.Name, string.Format("blob name should be {0}, and actully it's {1}", rootFiles[i].Name, blob.Name));
string localMd5 = Helper.GetFileContentMD5(Path.Combine(uploadDirRoot, rootFiles[i].Name));
Test.Assert(blob.BlobType == Microsoft.WindowsAzure.Storage.Blob.BlobType.BlockBlob, "blob type should be block blob");
Test.Assert(localMd5 == blob.Properties.ContentMD5, string.Format("blob content md5 should be {0}, and actualy it's {1}", localMd5, blob.Properties.ContentMD5));
}
}
finally
{
blobUtil.RemoveContainer(containerName);
}
}
public static void ClearContainer(CloudBlobContainer container, string prefix)
{
foreach (CloudBlockBlob blob in container.ListBlobs(useFlatBlobListing: true).Where(b => b is CloudBlockBlob).Select(b => b as CloudBlockBlob))
{
if (blob.Name.StartsWith(prefix))
{
Console.WriteLine("Deleting blob '" + blob.Uri.ToString() + "'");
blob.Delete();
}
}
}
private static void TestAccess(BlobContainerPublicAccessType accessType, CloudBlobContainer container, CloudBlob inputBlob)
{
StorageCredentials credentials = new StorageCredentials();
container = new CloudBlobContainer(container.Uri, credentials);
CloudPageBlob blob = new CloudPageBlob(inputBlob.Uri, credentials);
if (accessType.Equals(BlobContainerPublicAccessType.Container))
{
blob.FetchAttributes();
container.ListBlobs().ToArray();
container.FetchAttributes();
}
else if (accessType.Equals(BlobContainerPublicAccessType.Blob))
{
blob.FetchAttributes();
TestHelper.ExpectedException(
() => container.ListBlobs().ToArray(),
"List blobs while public access does not allow for listing",
HttpStatusCode.NotFound);
TestHelper.ExpectedException(
() => container.FetchAttributes(),
"Fetch container attributes while public access does not allow",
HttpStatusCode.NotFound);
}
else
{
TestHelper.ExpectedException(
() => blob.FetchAttributes(),
"Fetch blob attributes while public access does not allow",
HttpStatusCode.NotFound);
TestHelper.ExpectedException(
() => container.ListBlobs().ToArray(),
"List blobs while public access does not allow for listing",
HttpStatusCode.NotFound);
TestHelper.ExpectedException(
() => container.FetchAttributes(),
"Fetch container attributes while public access does not allow",
HttpStatusCode.NotFound);
}
}
// Access multiple blob at a time
public List <string> DownloadBlobList(string UserId, string UserTable = null)
{
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = GetCloudBlobContainer(UserId);
List <string> blobs = new List <string>();
if (UserTable != null)
{
foreach (var blobItem in container.ListBlobs(UserTable))
{
blobs.Add(Convert.ToString(blobItem.Uri));
}
}
else
{
foreach (var blobItem in container.ListBlobs(UserId))
{
blobs.Add(Convert.ToString(blobItem.Uri));
}
}
return(blobs);
}
/// <summary>
/// Creates a new asset and copies blobs from the specifed storage account.
/// </summary>
/// <param name="mediaBlobContainer">The specified blob container.</param>
/// <returns>The new asset.</returns>
public static IAsset CreateAssetFromExistingBlobs(CloudBlobContainer mediaBlobContainer)
{
// Create a new asset.
IAsset asset = _context.Assets.Create("NewAsset_" + Guid.NewGuid(), AssetCreationOptions.None);
IAccessPolicy writePolicy = _context.AccessPolicies.Create("writePolicy",
TimeSpan.FromHours(24), AccessPermissions.Write);
ILocator destinationLocator = _context.Locators.CreateLocator(LocatorType.Sas, asset, writePolicy);
CloudBlobClient destBlobStorage = _destinationStorageAccount.CreateCloudBlobClient();
// Get the asset container URI and Blob copy from mediaContainer to assetContainer.
string destinationContainerName = (new Uri(destinationLocator.Path)).Segments[1];
CloudBlobContainer assetContainer =
destBlobStorage.GetContainerReference(destinationContainerName);
if (assetContainer.CreateIfNotExists())
{
assetContainer.SetPermissions(new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Blob
});
}
var blobList = mediaBlobContainer.ListBlobs();
foreach (var sourceBlob in blobList)
{
var assetFile = asset.AssetFiles.Create((sourceBlob as ICloudBlob).Name);
CopyBlob(sourceBlob as ICloudBlob, assetContainer);
assetFile.ContentFileSize = (sourceBlob as ICloudBlob).Properties.Length;
assetFile.Update();
}
asset.Update();
destinationLocator.Delete();
writePolicy.Delete();
// Since we copied a set of Smooth Streaming files,
// set the .ism file to be the primary file.
// If we, for example, copied an .mp4, then the mp4 would be the primary file.
SetISMFileAsPrimary(asset);
return asset;
}
public static Tuple <bool, List <string> > listHubs(string account, string accountKey, string orgID, string studyID)
{
try
{
CloudStorageAccount storageAccount = new CloudStorageAccount(new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(account, accountKey), true);
Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer storageContainer = blobClient.GetContainerReference(AzureConfigContainerName);
List <String> hubList = lsDirectory(storageContainer.ListBlobs(), "/" + orgID + "/" + studyID + "/");
return(new Tuple <bool, List <string> >(true, hubList));
}
catch (Exception e)
{
return(new Tuple <bool, List <string> >(false, new List <string>()
{
e.Message
}));
}
}
public static void ListAllBlobs(CloudBlobContainer container)
{
foreach (IListBlobItem item in container.ListBlobs(null, false))
{
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob blob = (CloudBlockBlob)item;
Console.WriteLine("Block blob of length { 0}: { 1}", blob.Properties.Length,blob.Uri);
}
else if (item.GetType() == typeof(CloudPageBlob))
{
CloudPageBlob pageBlob = (CloudPageBlob)item;
Console.WriteLine("Page blob of length { 0}: { 1}", pageBlob.Properties.Length,pageBlob.Uri);
}
else if (item.GetType() == typeof(CloudBlobDirectory))
{
CloudBlobDirectory directory = (CloudBlobDirectory)item;
Console.WriteLine("Directory: { 0}", directory.Uri);
}
}
}
private static void DumpContainer(CloudBlobContainer container)
{
IEnumerable<IListBlobItem> list = container.ListBlobs();
IEnumerator<IListBlobItem> enumerator = list.GetEnumerator();
while (enumerator.MoveNext())
{
IListBlobItem item = (IListBlobItem)enumerator.Current;
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob blob = (CloudBlockBlob)item;
Console.WriteLine("Block blob of Length: {0}", blob.Properties.Length);
Console.WriteLine("Uri: {0}", blob.Uri);
}
else
{
Console.WriteLine("Unknown blob: ", item.ToString());
}
}
Console.WriteLine();
}
public void SetBlobContentWithInvalidBlobName()
{
string containerName = Utility.GenNameString("container");
StorageBlob.CloudBlobContainer container = blobUtil.CreateContainer(containerName);
try
{
int MaxBlobNameLength = 1024;
string blobName = new string('a', MaxBlobNameLength + 1);
List <StorageBlob.IListBlobItem> blobLists = container.ListBlobs(string.Empty, true, StorageBlob.BlobListingDetails.All).ToList();
Test.Assert(blobLists.Count == 0, string.Format("container {0} should contain {1} blobs, and actually it contain {2} blobs", containerName, 0, blobLists.Count));
Test.Assert(!agent.SetAzureStorageBlobContent(Path.Combine(uploadDirRoot, files[0]), containerName, StorageBlob.BlobType.BlockBlob, blobName), "upload blob with invalid blob name should be failed");
string expectedErrorMessage = string.Format("Blob name '{0}' is invalid.", blobName);
ExpectedStartsWithErrorMessage(expectedErrorMessage);
}
finally
{
blobUtil.RemoveContainer(containerName);
}
}
public override LogItem GetLogItem(string baseAddress, string path)
{
if (_sasUrl == null)
{
return null;
}
if (String.IsNullOrWhiteSpace(path))
{
path = String.Empty;
}
path = path.TrimStart('/');
var blobContainer = new CloudBlobContainer(new Uri(_sasUrl));
var name = Path.GetFileName(path.TrimEnd('/').Replace("/", "\\"));
if (path.EndsWith("/") || path == String.Empty)
{
var logItems = new List<LogItem>();
var logDirectory = new LogItems()
{
Url = baseAddress,
Name = name,
IsDirectory = true,
Path = path,
Root = Name,
Items = logItems
};
var blobs = blobContainer.ListBlobs(String.IsNullOrEmpty(path) ? null : path);
foreach (var blob in blobs)
{
if (blob is CloudBlockBlob)
{
var cloudBlockBlob = blob as CloudBlockBlob;
var innerName = Path.GetFileName(cloudBlockBlob.Name.TrimEnd('/').Replace("/", "\\"));
logItems.Add(new LogItem()
{
Name = innerName,
Size = cloudBlockBlob.Properties.Length,
Path = path + innerName,
Date = cloudBlockBlob.Properties.LastModified.HasValue ? (DateTime?)cloudBlockBlob.Properties.LastModified.Value.DateTime : null,
Url = baseAddress + innerName,
DownloadUrl = baseAddress + innerName + "&download=true"
});
}
else if (blob is CloudBlobDirectory)
{
var innerCloudBlobDirectory = blob as CloudBlobDirectory;
var innerPath = innerCloudBlobDirectory.Prefix;
var innerName = Path.GetFileName(innerPath.TrimEnd('/').Replace("/", "\\"));
logItems.Add(new LogItem()
{
IsDirectory = true,
Name = innerName,
Path = innerPath,
Url = baseAddress + innerName + "/"
});
}
}
return logDirectory;
}
var blockBlobReference = blobContainer.GetBlockBlobReference(path);
if (blockBlobReference.Exists())
{
return new LogItem()
{
Name = name,
Size = blockBlobReference.Properties.Length,
Path = path,
Date = blockBlobReference.Properties.LastModified.HasValue ? (DateTime?)blockBlobReference.Properties.LastModified.Value.DateTime : null,
Url = baseAddress,
DownloadUrl = blockBlobReference.Uri + _sas
};
}
return null;
}
private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlobContainer destinationContainer)
{
Trace.TraceInformation("Getting list of inactive packages.");
IList<string> packageIds;
using (var connection = await _sourceDatabase.ConnectTo())
{
var sql = "[dbo].[DownloadReportListInactive]";
packageIds = (await connection.QueryAsync<string>(sql, CommandType.StoredProcedure)).ToList();
}
Trace.TraceInformation("Found {0} inactive packages.", packageIds.Count);
// Collect the list of reports
var subContainer = "recentpopularity/";
Trace.TraceInformation("Collecting list of package detail reports");
var reports = destinationContainer.ListBlobs(subContainer + _recentPopularityDetailByPackageReportBaseName)
.OfType<CloudBlockBlob>()
.Select(b => b.Name);
var reportSet = new HashSet<string>(reports);
Trace.TraceInformation("Collected {0} package detail reports", reportSet.Count);
Parallel.ForEach(packageIds, new ParallelOptions { MaxDegreeOfParallelism = 8}, async id =>
{
string reportName = _recentPopularityDetailByPackageReportBaseName + id;
string blobName = subContainer + reportName + ".json";
if (reportSet.Contains(blobName))
{
var blob = destinationContainer.GetBlockBlobReference(blobName);
Trace.TraceInformation("{0}: Deleting empty report from {1}", reportName, blob.Uri.AbsoluteUri);
await blob.DeleteIfExistsAsync();
Trace.TraceInformation("{0}: Deleted empty report from {1}", reportName, blob.Uri.AbsoluteUri);
}
});
}
public void CloudBlobSnapshot()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
// Upload some data to the blob.
MemoryStream originalData = new MemoryStream(GetRandomBuffer(1024));
CloudAppendBlob appendBlob = container.GetAppendBlobReference(BlobName);
appendBlob.CreateOrReplace();
appendBlob.AppendBlock(originalData, null);
CloudBlob blob = container.GetBlobReference(BlobName);
blob.FetchAttributes();
Assert.IsFalse(blob.IsSnapshot);
Assert.IsNull(blob.SnapshotTime, "Root blob has SnapshotTime set");
Assert.IsFalse(blob.SnapshotQualifiedUri.Query.Contains("snapshot"));
Assert.AreEqual(blob.Uri, blob.SnapshotQualifiedUri);
CloudBlob snapshot1 = blob.Snapshot();
Assert.AreEqual(blob.Properties.ETag, snapshot1.Properties.ETag);
Assert.AreEqual(blob.Properties.LastModified, snapshot1.Properties.LastModified);
Assert.IsTrue(snapshot1.IsSnapshot);
Assert.IsNotNull(snapshot1.SnapshotTime, "Snapshot does not have SnapshotTime set");
Assert.AreEqual(blob.Uri, snapshot1.Uri);
Assert.AreNotEqual(blob.SnapshotQualifiedUri, snapshot1.SnapshotQualifiedUri);
Assert.AreNotEqual(snapshot1.Uri, snapshot1.SnapshotQualifiedUri);
Assert.IsTrue(snapshot1.SnapshotQualifiedUri.Query.Contains("snapshot"));
CloudBlob snapshot2 = blob.Snapshot();
Assert.IsTrue(snapshot2.SnapshotTime.Value > snapshot1.SnapshotTime.Value);
snapshot1.FetchAttributes();
snapshot2.FetchAttributes();
blob.FetchAttributes();
AssertAreEqual(snapshot1.Properties, blob.Properties, false);
CloudBlob snapshot1Clone = new CloudBlob(new Uri(blob.Uri + "?snapshot=" + snapshot1.SnapshotTime.Value.ToString("O")), blob.ServiceClient.Credentials);
Assert.IsNotNull(snapshot1Clone.SnapshotTime, "Snapshot clone does not have SnapshotTime set");
Assert.AreEqual(snapshot1.SnapshotTime.Value, snapshot1Clone.SnapshotTime.Value);
snapshot1Clone.FetchAttributes();
AssertAreEqual(snapshot1.Properties, snapshot1Clone.Properties, false);
CloudBlob snapshotCopy = container.GetBlobReference("blob2");
snapshotCopy.StartCopy(TestHelper.Defiddler(snapshot1.Uri));
WaitForCopy(snapshotCopy);
Assert.AreEqual(CopyStatus.Success, snapshotCopy.CopyState.Status);
using (Stream snapshotStream = snapshot1.OpenRead())
{
snapshotStream.Seek(0, SeekOrigin.End);
TestHelper.AssertStreamsAreEqual(originalData, snapshotStream);
}
appendBlob.CreateOrReplace();
blob.FetchAttributes();
using (Stream snapshotStream = snapshot1.OpenRead())
{
snapshotStream.Seek(0, SeekOrigin.End);
TestHelper.AssertStreamsAreEqual(originalData, snapshotStream);
}
List <IListBlobItem> blobs = container.ListBlobs(null, true, BlobListingDetails.All, null, null).ToList();
Assert.AreEqual(4, blobs.Count);
AssertAreEqual(snapshot1, (CloudBlob)blobs[0]);
AssertAreEqual(snapshot2, (CloudBlob)blobs[1]);
AssertAreEqual(blob, (CloudBlob)blobs[2]);
AssertAreEqual(snapshotCopy, (CloudBlob)blobs[3]);
}
finally
{
container.DeleteIfExists();
}
}
public static IEnumerable<CloudBlobDirectory> ListSubdirectories(CloudBlobContainer cloudBlobContainer) {
var l = cloudBlobContainer.Uri.AbsolutePath.Length;
return (from blob in cloudBlobContainer.ListBlobs().Select(each => each.Uri.AbsolutePath.Substring(l + 1))
let i = blob.IndexOf('/')
where i > -1
select blob.Substring(0, i)).Distinct().Select(cloudBlobContainer.GetDirectoryReference);
}
public IEnumerable<IListBlobItem> ListBlobs(CloudBlobContainer container)
{
return container.ListBlobs(null, true, BlobListingDetails.All, null, null);
}
/// <summary>
/// Queries the BLOB.
/// </summary>
/// <param name="container">The container.</param>
/// <param name="contentType">Type of the content.</param>
/// <param name="md5">The MD5.</param>
/// <param name="length">The length.</param>
/// <param name="limitCount">The limit count.</param>
/// <returns>IEnumerable<CloudBlockBlob>.</returns>
public static IEnumerable<CloudBlockBlob> QueryBlob(CloudBlobContainer container, string contentType, string md5, long? length, int limitCount = 100)
{
try
{
container.CheckNullObject("container");
if (string.IsNullOrWhiteSpace(contentType) && string.IsNullOrWhiteSpace(md5) && length == null)
{
return container.ListBlobs().OfType<CloudBlockBlob>().Take(limitCount);
}
else
{
return container.ListBlobs(blobListingDetails: BlobListingDetails.Metadata)
.OfType<CloudBlockBlob>()
.Where(item =>
{
return ((string.IsNullOrWhiteSpace(contentType) ||
item.Properties.ContentType.Equals(contentType,
StringComparison.InvariantCultureIgnoreCase))
&&
(string.IsNullOrWhiteSpace(md5) ||
item.Properties.ContentMD5.Equals(md5,
StringComparison.InvariantCultureIgnoreCase))
&& (length == null || item.Properties.Length == length.Value));
}).Take(limitCount);
}
}
catch (Exception ex)
{
throw ex.Handle("QueryBlob", new { container = container == null ? null : container.Uri.ToString(), contentType, md5, length, limitCount });
}
}
public void CloudBlobSoftDeleteSnapshot()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
//Enables a delete retention policy on the blob with 1 day of default retention days
container.ServiceClient.EnableSoftDelete();
container.Create();
// Upload some data to the blob.
MemoryStream originalData = new MemoryStream(GetRandomBuffer(1024));
CloudAppendBlob appendBlob = container.GetAppendBlobReference(BlobName);
appendBlob.UploadFromStream(originalData);
CloudBlob blob = container.GetBlobReference(BlobName);
//create snapshot via api
CloudBlob snapshot = blob.Snapshot();
//create snapshot via write protection
appendBlob.UploadFromStream(originalData);
//we should have 2 snapshots 1 regular and 1 deleted: there is no way to get only the deleted snapshots but the below listing will get both snapshot types
int blobCount = 0;
int deletedSnapshotCount = 0;
int snapShotCount = 0;
foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.Snapshots | BlobListingDetails.Deleted))
{
CloudAppendBlob blobItem = (CloudAppendBlob)item;
Assert.AreEqual(blobItem.Name, BlobName);
if (blobItem.IsSnapshot)
{
snapShotCount++;
}
if (blobItem.IsDeleted)
{
Assert.IsNotNull(blobItem.Properties.DeletedTime);
Assert.AreEqual(blobItem.Properties.RemainingDaysBeforePermanentDelete, 0);
deletedSnapshotCount++;
}
blobCount++;
}
Assert.AreEqual(blobCount, 3);
Assert.AreEqual(deletedSnapshotCount, 1);
Assert.AreEqual(snapShotCount, 2);
blobCount = 0;
foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.Snapshots))
{
CloudAppendBlob blobItem = (CloudAppendBlob)item;
Assert.AreEqual(blobItem.Name, BlobName);
Assert.IsFalse(blobItem.IsDeleted);
Assert.IsNull(blobItem.Properties.DeletedTime);
Assert.IsNull(blobItem.Properties.RemainingDaysBeforePermanentDelete);
blobCount++;
}
Assert.AreEqual(blobCount, 2);
//Delete Blob and snapshots
blob.Delete(DeleteSnapshotsOption.IncludeSnapshots);
Assert.IsFalse(blob.Exists());
Assert.IsFalse(snapshot.Exists());
blobCount = 0;
foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All))
{
CloudAppendBlob blobItem = (CloudAppendBlob)item;
Assert.AreEqual(blobItem.Name, BlobName);
Assert.IsTrue(blobItem.IsDeleted);
Assert.IsNotNull(blobItem.Properties.DeletedTime);
Assert.AreEqual(blobItem.Properties.RemainingDaysBeforePermanentDelete, 0);
blobCount++;
}
Assert.AreEqual(blobCount, 3);
blob.Undelete();
blob.FetchAttributes();
Assert.IsFalse(blob.IsDeleted);
Assert.IsNull(blob.Properties.DeletedTime);
Assert.IsNull(blob.Properties.RemainingDaysBeforePermanentDelete);
blobCount = 0;
foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All))
{
CloudAppendBlob blobItem = (CloudAppendBlob)item;
Assert.AreEqual(blobItem.Name, BlobName);
Assert.IsFalse(blobItem.IsDeleted);
Assert.IsNull(blobItem.Properties.DeletedTime);
Assert.IsNull(blobItem.Properties.RemainingDaysBeforePermanentDelete);
blobCount++;
}
Assert.AreEqual(blobCount, 3);
}
finally
{
container.ServiceClient.DisableSoftDelete();
container.DeleteIfExists();
}
}
/// <summary>
/// Copy all blobs from <param name="srcContainer">srcContainer</param> to the
/// <param name="destContainer">destContainer</param>.
/// </summary>
/// <remarks>This method uses a Parallel ForEach to provide async copying for large files.</remarks>
/// <param name="srcContainer">The source Blob Container</param>
/// <param name="destContainer"></param>
private static void CopyBlobs(
CloudBlobContainer srcContainer,
CloudBlobContainer destContainer)
{
// Get source lists from Azure (LazyLoaded):
var srcBlobList
= srcContainer.ListBlobs(string.Empty, true, BlobListingDetails.All); // set to none in prod (4perf)
var destBlobList
= destContainer.ListBlobs(string.Empty, true, BlobListingDetails.All); // set to none in prod (4perf)
// Convert to IEnumerable to avoid multiple enumeration:
var srcBlobItems = srcBlobList as IList<IListBlobItem> ?? srcBlobList.ToList();
var destBlobItems = destBlobList as IList<IListBlobItem> ?? destBlobList.ToList();
// Run the core blob sort, filter, and copy method:
BlobFilter(srcBlobItems, destBlobItems);
// Exit Method:
return;
}
/// <summary>
/// Queries the BLOB.
/// </summary>
/// <param name="container">The container.</param>
/// <param name="filterFunction">The filter function.</param>
/// <param name="limitCount">The limit count.</param>
/// <returns>IEnumerable<CloudBlockBlob>.</returns>
public static IEnumerable<CloudBlockBlob> QueryBlob(CloudBlobContainer container, Func<CloudBlockBlob, bool> filterFunction, int? limitCount = null)
{
try
{
container.CheckNullObject("container");
if (filterFunction != null)
{
return limitCount == null ? container.ListBlobs().OfType<CloudBlockBlob>().Where(filterFunction) : container.ListBlobs().OfType<CloudBlockBlob>().Where(filterFunction).Take(limitCount.Value);
}
else
{
return limitCount == null ? container.ListBlobs().OfType<CloudBlockBlob>() : container.ListBlobs().OfType<CloudBlockBlob>().Take(limitCount.Value);
}
}
catch (Exception ex)
{
throw ex.Handle("QueryBlob", new { container = container == null ? null : container.Uri.ToString(), limitCount = limitCount });
}
}
public void CloudBlobSoftDeleteNoSnapshot()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
//Enables a delete retention policy on the blob with 1 day of default retention days
container.ServiceClient.EnableSoftDelete();
Thread.Sleep(15000);
container.Create();
// Upload some data to the blob.
MemoryStream originalData = new MemoryStream(GetRandomBuffer(1024));
CloudAppendBlob appendBlob = container.GetAppendBlobReference(BlobName);
appendBlob.CreateOrReplace();
appendBlob.AppendBlock(originalData, null);
CloudBlob blob = container.GetBlobReference(BlobName);
Assert.IsTrue(blob.Exists());
Assert.IsFalse(blob.IsDeleted);
blob.Delete();
Assert.IsFalse(blob.Exists());
int blobCount = 0;
foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All))
{
CloudAppendBlob blobItem = (CloudAppendBlob)item;
Assert.AreEqual(blobItem.Name, BlobName);
Assert.IsTrue(blobItem.IsDeleted);
Assert.IsNotNull(blobItem.Properties.DeletedTime);
Assert.AreEqual(blobItem.Properties.RemainingDaysBeforePermanentDelete, 0);
blobCount++;
}
Assert.AreEqual(blobCount, 1);
blob.Undelete();
blob.FetchAttributes();
Assert.IsFalse(blob.IsDeleted);
Assert.IsNull(blob.Properties.DeletedTime);
Assert.IsNull(blob.Properties.RemainingDaysBeforePermanentDelete);
blobCount = 0;
foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All))
{
CloudAppendBlob blobItem = (CloudAppendBlob)item;
Assert.AreEqual(blobItem.Name, BlobName);
Assert.IsFalse(blobItem.IsDeleted);
Assert.IsNull(blobItem.Properties.DeletedTime);
Assert.IsNull(blobItem.Properties.RemainingDaysBeforePermanentDelete);
blobCount++;
}
Assert.AreEqual(blobCount, 1);
}
finally
{
container.ServiceClient.DisableSoftDelete();
container.DeleteIfExists();
}
}
private IEnumerable<string> BlobNamesInContainer(CloudBlobContainer container, string prefix = null)
{
return container.ListBlobs(prefix, true)
.OfType<ICloudBlob>()
.Where(b => !string.IsNullOrWhiteSpace(b.Name))
.Select(blockBlob => blockBlob.Name);
}
/// <summary>
/// List all blobs in specified containers
/// </summary>
/// <param name="container">A cloudblobcontainer object</param>
/// <param name="prefix">Blob prefix</param>
/// <param name="useFlatBlobListing">Use flat blob listing(whether treat "container/" as directory)</param>
/// <param name="blobListingDetails">Blob listing details</param>
/// <param name="options">Blob request option</param>
/// <param name="operationContext">Operation context</param>
/// <returns>An enumerable collection of icloudblob</returns>
public IEnumerable<IListBlobItem> ListBlobs(CloudBlobContainer container, string prefix, bool useFlatBlobListing, BlobListingDetails blobListingDetails, BlobRequestOptions options, OperationContext operationContext)
{
return container.ListBlobs(prefix, useFlatBlobListing, blobListingDetails, options, operationContext);
}
/// <summary>
/// Tests a container SAS to determine which operations it allows.
/// </summary>
/// <param name="sasUri">A string containing a URI with a SAS appended.</param>
/// <param name="blobName">A string containing the name of the blob.</param>
/// <param name="blobContent">A string content content to write to the blob.</param>
/// <returns>A Task object.</returns>
private static async Task TestContainerSASAsync(string sasUri, string blobName, string blobContent)
{
// Try performing container operations with the SAS provided.
// Note that the storage account credentials are not required here; the SAS provides the necessary
// authentication information on the URI.
// Return a reference to the container using the SAS URI.
CloudBlobContainer container = new CloudBlobContainer(new Uri(sasUri));
// Return a reference to a blob to be created in the container.
CloudBlockBlob blob = container.GetBlockBlobReference(blobName);
// Write operation: Upload a new blob to the container.
try
{
MemoryStream msWrite = new MemoryStream(Encoding.UTF8.GetBytes(blobContent));
msWrite.Position = 0;
using (msWrite)
{
await blob.UploadFromStreamAsync(msWrite);
}
Console.WriteLine("Write operation succeeded for SAS {0}", sasUri);
Console.WriteLine();
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode == 403)
{
Console.WriteLine("Write operation failed for SAS {0}", sasUri);
Console.WriteLine("Additional error information: " + e.Message);
Console.WriteLine();
}
else
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
// List operation: List the blobs in the container.
try
{
foreach (ICloudBlob blobItem in container.ListBlobs(
prefix: null,
useFlatBlobListing: true,
blobListingDetails: BlobListingDetails.None,
options: null,
operationContext: null))
{
Console.WriteLine(blobItem.Uri);
}
Console.WriteLine("List operation succeeded for SAS {0}", sasUri);
Console.WriteLine();
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode == 403)
{
Console.WriteLine("List operation failed for SAS {0}", sasUri);
Console.WriteLine("Additional error information: " + e.Message);
Console.WriteLine();
}
else
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
// Read operation: Read the contents of the blob we created above.
try
{
MemoryStream msRead = new MemoryStream();
msRead.Position = 0;
using (msRead)
{
await blob.DownloadToStreamAsync(msRead);
Console.WriteLine(msRead.Length);
}
Console.WriteLine();
Console.WriteLine("Read operation succeeded for SAS {0}", sasUri);
Console.WriteLine();
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode == 403)
{
Console.WriteLine("Read operation failed for SAS {0}", sasUri);
Console.WriteLine("Additional error information: " + e.Message);
Console.WriteLine();
}
else
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
Console.WriteLine();
// Delete operation: Delete the blob we created above.
try
{
await blob.DeleteAsync();
Console.WriteLine("Delete operation succeeded for SAS {0}", sasUri);
Console.WriteLine();
}
catch (StorageException e)
{
if (e.RequestInformation.HttpStatusCode == 403)
{
Console.WriteLine("Delete operation failed for SAS {0}", sasUri);
Console.WriteLine("Additional error information: " + e.Message);
Console.WriteLine();
}
else
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
}
Console.WriteLine();
}
/// <summary>
/// Monitors a blob copy operation
/// </summary>
/// <param name="destContainer">The destination container to monitor</param>
/// <param name="log">The log to write output</param>
/// <returns></returns>
public async Task MonitorCopy(CloudBlobContainer destContainer, string fileName, TextWriter log)
{
bool pendingCopy = true;
int waitSeconds = 3;
while (pendingCopy)
{
pendingCopy = false;
//This is going to get all of the blobs in the container.
var destBlobList = destContainer.ListBlobs(prefix:fileName, useFlatBlobListing: true , blobListingDetails: BlobListingDetails.Copy);
foreach (var dest in destBlobList)
{
var destBlob = dest as CloudBlob;
if (destBlob.CopyState != null)
{
if (destBlob.CopyState.Status == CopyStatus.Aborted ||
destBlob.CopyState.Status == CopyStatus.Failed)
{
// Log the copy status description for diagnostics
// and restart copy
await log.WriteLineAsync(destBlob.CopyState.ToString());
pendingCopy = false;
//Could restart the operation here if it fails
pendingCopy = true;
try
{
destBlob.StartCopy(destBlob.CopyState.Source);
}
catch (Exception oops)
{
await log.WriteLineAsync(oops.Message);
throw oops;
}
}
else if (destBlob.CopyState.Status == CopyStatus.Pending)
{
// We need to continue waiting for this pending copy
// However, let us log copy state for diagnostics
await log.WriteLineAsync(destBlob.CopyState.ToString());
pendingCopy = true;
}
}
else
{
//What the hell does this mean? Why are we getting Null for this query?!?
// else we completed this pending copy
await log.WriteLineAsync(destBlob.Name.ToString() + " is null?!?!");
// does this mean it is pending? wtf?
}
pendingCopy = false;
}
//Wait number of milliseconds before trying again
Thread.Sleep(waitSeconds * 1000);
};
}
public void CloudBlobSoftDeleteNoSnapshotAPM()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
//Enables a delete retention policy on the blob with 1 day of default retention days
container.ServiceClient.EnableSoftDelete();
container.Create();
// Upload some data to the blob.
MemoryStream originalData = new MemoryStream(GetRandomBuffer(1024));
CloudAppendBlob appendBlob = container.GetAppendBlobReference(BlobName);
appendBlob.CreateOrReplace();
appendBlob.AppendBlock(originalData, null);
CloudBlob blob = container.GetBlobReference(BlobName);
Assert.IsFalse(blob.IsDeleted);
IAsyncResult result;
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
result = blob.BeginDelete(DeleteSnapshotsOption.None, null, null, null, ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blob.EndDelete(result);
}
Assert.IsFalse(blob.Exists());
int blobCount = 0;
foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All))
{
CloudAppendBlob blobItem = (CloudAppendBlob)item;
Assert.AreEqual(blobItem.Name, BlobName);
Assert.IsTrue(blobItem.IsDeleted);
Assert.IsNotNull(blobItem.Properties.DeletedTime);
Assert.AreEqual(blobItem.Properties.RemainingDaysBeforePermanentDelete, 0);
blobCount++;
}
Assert.AreEqual(blobCount, 1);
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
result = blob.BeginUndelete(ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blob.EndUndelete(result);
}
blob.FetchAttributes();
Assert.IsFalse(blob.IsDeleted);
Assert.IsNull(blob.Properties.DeletedTime);
Assert.IsNull(blob.Properties.RemainingDaysBeforePermanentDelete);
blobCount = 0;
foreach (IListBlobItem item in container.ListBlobs(null, true, BlobListingDetails.All))
{
CloudAppendBlob blobItem = (CloudAppendBlob)item;
Assert.AreEqual(blobItem.Name, BlobName);
Assert.IsFalse(blobItem.IsDeleted);
Assert.IsNull(blobItem.Properties.DeletedTime);
Assert.IsNull(blobItem.Properties.RemainingDaysBeforePermanentDelete);
blobCount++;
}
Assert.AreEqual(blobCount, 1);
}
finally
{
container.ServiceClient.DisableSoftDelete();
container.DeleteIfExists();
}
}
/// <summary>
/// Returns a flattened list of <see cref="CloudBlockBlob"/> items from the specified <paramref name="CloudBlobContainer"/>.
/// </summary>
/// <param name="container"></param>
/// <returns></returns>
private static List<CloudBlockBlob> GetBlobList(CloudBlobContainer container)
{
var blobs = container
.ListBlobs(useFlatBlobListing: true, blobListingDetails: BlobListingDetails.Metadata)
.OfType<CloudBlockBlob>()
.ToList();
return blobs;
}
public static void CopyBlobs(
CloudBlobContainer srcContainer,
string policyId,
CloudBlobContainer destContainer)
{
// get the SAS token to use for all blobs
string blobToken = srcContainer.GetSharedAccessSignature(
new SharedAccessBlobPolicy(), policyId);
var srcBlobList = srcContainer.ListBlobs(null, false);
foreach (var src in srcBlobList)
{
var srcBlob = src as CloudBlob;
try {
// Create appropriate destination blob type to match the source blob
CloudBlob destBlob;
if (srcBlob.Properties.BlobType == BlobType.BlockBlob)
{
destBlob = destContainer.GetBlockBlobReference(srcBlob.Name);
}
else
{
destBlob = destContainer.GetPageBlobReference(srcBlob.Name);
}
// copy using src blob as SAS
// destBlob.StartCopyFromBlob(new Uri(srcBlob.Uri.AbsoluteUri + blobToken));
destBlob.StartCopy(new Uri(srcBlob.Uri.AbsoluteUri));
Console.WriteLine("Copied: " + srcBlob.Uri.AbsoluteUri);
}
catch (Exception e)
{
Console.WriteLine("Error: " + e.Message);
}
}
}
/// <summary>
/// Basic operations to work with block blobs
/// </summary>
/// <returns>Task<returns>
private static async Task BasicStorageBlockBlobOperationsWithAccountSASAsync()
{
const string imageToUpload = "HelloWorld.png";
string blockBlobContainerName = "demoblockblobcontainer-" + Guid.NewGuid();
// Call GetAccountSASToken to get a sasToken based on the Storage Account Key
string sasToken = GetAccountSASToken();
// Create an AccountSAS from the SASToken
StorageCredentials accountSAS = new StorageCredentials(sasToken);
//Informational: Print the Account SAS Signature and Token
Console.WriteLine();
Console.WriteLine("Account SAS Signature: " + accountSAS.SASSignature);
Console.WriteLine("Account SAS Token: " + accountSAS.SASToken);
Console.WriteLine();
// Create a container for organizing blobs within the storage account.
Console.WriteLine("1. Creating Container using Account SAS");
// Get the Container Uri by passing the Storage Account and the container Name
Uri ContainerUri = GetContainerSASUri(blockBlobContainerName);
// Create a CloudBlobContainer by using the Uri and the sasToken
CloudBlobContainer container = new CloudBlobContainer(ContainerUri, new StorageCredentials(sasToken));
try
{
await container.CreateIfNotExistsAsync();
}
catch (StorageException)
{
Console.WriteLine("If you are running with the default configuration please make sure you have started the storage emulator. Press the Windows key and type Azure Storage to select and run it from the list of applications - then restart the sample.");
Console.ReadLine();
throw;
}
// To view the uploaded blob in a browser, you have two options. The first option is to use a Shared Access Signature (SAS) token to delegate
// access to the resource. See the documentation links at the top for more information on SAS. The second approach is to set permissions
// to allow public access to blobs in this container. Uncomment the line below to use this approach. Then you can view the image
// using: https://[InsertYourStorageAccountNameHere].blob.core.windows.net/democontainer/HelloWorld.png
// await container.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob });
// Upload a BlockBlob to the newly created container
Console.WriteLine("2. Uploading BlockBlob");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(imageToUpload);
await blockBlob.UploadFromFileAsync(imageToUpload, FileMode.Open);
// List all the blobs in the container
Console.WriteLine("3. List Blobs in Container");
foreach (IListBlobItem blob in container.ListBlobs())
{
// Blob type will be CloudBlockBlob, CloudPageBlob or CloudBlobDirectory
// Use blob.GetType() and cast to appropriate type to gain access to properties specific to each type
Console.WriteLine("- {0} (type: {1})", blob.Uri, blob.GetType());
}
// Download a blob to your file system
Console.WriteLine("4. Download Blob from {0}", blockBlob.Uri.AbsoluteUri);
await blockBlob.DownloadToFileAsync(string.Format("./CopyOf{0}", imageToUpload), FileMode.Create);
// Create a read-only snapshot of the blob
Console.WriteLine("5. Create a read-only snapshot of the blob");
CloudBlockBlob blockBlobSnapshot = await blockBlob.CreateSnapshotAsync(null, null, null, null);
// Clean up after the demo
Console.WriteLine("6. Delete block Blob and all of its snapshots");
await blockBlob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, null, null);
Console.WriteLine("7. Delete Container");
await container.DeleteIfExistsAsync();
}
public void CloudBlobSnapshotAPM()
{
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
MemoryStream originalData = new MemoryStream(GetRandomBuffer(1024));
CloudAppendBlob appendBlob = container.GetAppendBlobReference(BlobName);
appendBlob.CreateOrReplace();
appendBlob.AppendBlock(originalData, null);
CloudBlob blob = container.GetBlobReference(BlobName);
blob.FetchAttributes();
IAsyncResult result;
using (AutoResetEvent waitHandle = new AutoResetEvent(false))
{
result = blob.BeginSnapshot(ar => waitHandle.Set(), null);
waitHandle.WaitOne();
CloudBlob snapshot1 = blob.EndSnapshot(result);
Assert.AreEqual(blob.Properties.ETag, snapshot1.Properties.ETag);
Assert.AreEqual(blob.Properties.LastModified, snapshot1.Properties.LastModified);
Assert.IsTrue(snapshot1.IsSnapshot);
Assert.IsNotNull(snapshot1.SnapshotTime, "Snapshot does not have SnapshotTime set");
Assert.AreEqual(blob.Uri, snapshot1.Uri);
Assert.AreNotEqual(blob.SnapshotQualifiedUri, snapshot1.SnapshotQualifiedUri);
Assert.AreNotEqual(snapshot1.Uri, snapshot1.SnapshotQualifiedUri);
Assert.IsTrue(snapshot1.SnapshotQualifiedUri.Query.Contains("snapshot"));
result = blob.BeginSnapshot(ar => waitHandle.Set(), null);
waitHandle.WaitOne();
CloudBlob snapshot2 = blob.EndSnapshot(result);
Assert.IsTrue(snapshot2.SnapshotTime.Value > snapshot1.SnapshotTime.Value);
snapshot1.FetchAttributes();
snapshot2.FetchAttributes();
blob.FetchAttributes();
AssertAreEqual(snapshot1.Properties, blob.Properties);
CloudBlob snapshotCopy = container.GetBlobReference("blob2");
result = snapshotCopy.BeginStartCopy(snapshot1.Uri, null, null, null, null, ar => waitHandle.Set(), null);
waitHandle.WaitOne();
snapshotCopy.EndStartCopy(result);
WaitForCopy(snapshotCopy);
Assert.AreEqual(CopyStatus.Success, snapshotCopy.CopyState.Status);
result = snapshot1.BeginOpenRead(ar => waitHandle.Set(), null);
waitHandle.WaitOne();
using (Stream snapshotStream = snapshot1.EndOpenRead(result))
{
snapshotStream.Seek(0, SeekOrigin.End);
TestHelper.AssertStreamsAreEqual(originalData, snapshotStream);
}
result = appendBlob.BeginCreateOrReplace(ar => waitHandle.Set(), null);
waitHandle.WaitOne();
appendBlob.EndCreateOrReplace(result);
result = blob.BeginFetchAttributes(ar => waitHandle.Set(), null);
waitHandle.WaitOne();
blob.EndFetchAttributes(result);
result = snapshot1.BeginOpenRead(ar => waitHandle.Set(), null);
waitHandle.WaitOne();
using (Stream snapshotStream = snapshot1.EndOpenRead(result))
{
snapshotStream.Seek(0, SeekOrigin.End);
TestHelper.AssertStreamsAreEqual(originalData, snapshotStream);
}
List <IListBlobItem> blobs = container.ListBlobs(null, true, BlobListingDetails.All, null, null).ToList();
Assert.AreEqual(4, blobs.Count);
AssertAreEqual(snapshot1, (CloudBlob)blobs[0]);
AssertAreEqual(snapshot2, (CloudBlob)blobs[1]);
AssertAreEqual(blob, (CloudBlob)blobs[2]);
AssertAreEqual(snapshotCopy, (CloudBlob)blobs[3]);
}
}
finally
{
container.DeleteIfExists();
}
}
private static void DownloadTestCredentials(string testEnvironment, string downloadDirectoryPath, string blobUri, string storageAccount, string storageKey)
{
string containerPath = string.Format(EnvironmentPathFormat, testEnvironment);
StorageCredentials credentials = new StorageCredentials(storageAccount, storageKey);
CloudBlobClient blobClient = new CloudBlobClient(new Uri(blobUri), credentials);
blobContainer = blobClient.GetContainerReference(containerPath);
foreach (IListBlobItem blobItem in blobContainer.ListBlobs())
{
ICloudBlob blob = blobClient.GetBlobReferenceFromServer(blobItem.Uri);
Console.WriteLine("Downloading file {0} from blob Uri {1}", blob.Name, blob.Uri);
FileStream blobStream = new FileStream(Path.Combine(downloadDirectoryPath, blob.Name), FileMode.Create);
blob.DownloadToStream(blobStream);
blobStream.Flush();
blobStream.Close();
}
}
public void CloudBlobClientListBlobsSegmentedWithEmptyPrefix()
{
string name = "bb" + GetRandomContainerName();
CloudBlobClient blobClient = GenerateCloudBlobClient();
CloudBlobContainer rootContainer = blobClient.GetRootContainerReference();
CloudBlobContainer container = blobClient.GetContainerReference(name);
try
{
rootContainer.CreateIfNotExists();
container.Create();
List <Uri> preExistingBlobs = rootContainer.ListBlobs().Select(b => b.Uri).ToList();
List <string> blobNames = CreateBlobs(container, 3, BlobType.BlockBlob);
List <string> rootBlobNames = CreateBlobs(rootContainer, 2, BlobType.BlockBlob);
BlobResultSegment results;
BlobContinuationToken token = null;
List <Uri> listedBlobs = new List <Uri>();
do
{
results = blobClient.ListBlobsSegmented("", token);
token = results.ContinuationToken;
foreach (IListBlobItem blob in results.Results)
{
if (preExistingBlobs.Contains(blob.Uri))
{
continue;
}
else
{
if (blob is CloudPageBlob)
{
((CloudPageBlob)blob).Delete();
}
else
{
((CloudBlockBlob)blob).Delete();
}
listedBlobs.Add(blob.Uri);
}
}
}while (token != null);
Assert.AreEqual(2, listedBlobs.Count);
do
{
results = container.ListBlobsSegmented("", false, BlobListingDetails.None, null, token, null, null);
token = results.ContinuationToken;
foreach (IListBlobItem blob in results.Results)
{
if (preExistingBlobs.Contains(blob.Uri))
{
continue;
}
else
{
if (blob is CloudPageBlob)
{
((CloudPageBlob)blob).Delete();
}
else
{
((CloudBlockBlob)blob).Delete();
}
listedBlobs.Add(blob.Uri);
}
}
}while (token != null);
Assert.AreEqual(5, listedBlobs.Count);
}
finally
{
container.DeleteIfExists();
}
}
/// <summary>
/// Gets a reference to a blob created previously, and copies it to a new blob in the same container.
/// </summary>
/// <param name="container">A CloudBlobContainer object.</param>
/// <returns>A Task object.</returns>
private static async Task CopyBlockBlobAsync(CloudBlobContainer container)
{
CloudBlockBlob sourceBlob = null;
CloudBlockBlob destBlob = null;
string leaseId = null;
try
{
// Get a block blob from the container to use as the source.
sourceBlob = container.ListBlobs().OfType<CloudBlockBlob>().FirstOrDefault();
// Lease the source blob for the copy operation to prevent another client from modifying it.
// Specifying null for the lease interval creates an infinite lease.
leaseId = await sourceBlob.AcquireLeaseAsync(null);
// Get a reference to a destination blob (in this case, a new blob).
destBlob = container.GetBlockBlobReference("copy of " + sourceBlob.Name);
// Ensure that the source blob exists.
if (await sourceBlob.ExistsAsync())
{
// Get the ID of the copy operation.
string copyId = await destBlob.StartCopyAsync(sourceBlob);
// Fetch the destination blob's properties before checking the copy state.
await destBlob.FetchAttributesAsync();
Console.WriteLine("Status of copy operation: {0}", destBlob.CopyState.Status);
Console.WriteLine("Completion time: {0}", destBlob.CopyState.CompletionTime);
Console.WriteLine("Bytes copied: {0}", destBlob.CopyState.BytesCopied.ToString());
Console.WriteLine("Total bytes: {0}", destBlob.CopyState.TotalBytes.ToString());
Console.WriteLine();
}
}
catch (StorageException e)
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
finally
{
// Break the lease on the source blob.
if (sourceBlob != null)
{
await sourceBlob.FetchAttributesAsync();
if (sourceBlob.Properties.LeaseState != LeaseState.Available)
{
await sourceBlob.BreakLeaseAsync(new TimeSpan(0));
}
}
}
}
/// <summary>
/// Given the shared access signature of a container generates a set blob SaS's
/// </summary>
public void CopyBlobsFromContainerSas(string containerSas, string destinationContainer)
{
var container = new CloudBlobContainer(new Uri(containerSas));
var blobs = container.ListBlobs("blob/2014/08/07", true);
var destinationAccountStorageCredentials = new StorageCredentials(AccountName, AccountKey);
var destinationAccount = new CloudStorageAccount(destinationAccountStorageCredentials, true);
var destinationBlobClient = destinationAccount.CreateCloudBlobClient();
var destinationContainerInner = destinationBlobClient.GetContainerReference(destinationContainer);
foreach (var blob in blobs)
{
var blockBlob = blob as CloudBlockBlob;
using(var reader = new StreamReader(blockBlob.OpenRead()))
{
var destinationBlob = destinationContainerInner.GetBlockBlobReference(blockBlob.Name);
string contents = reader.ReadToEnd();
using (var writer = new StreamWriter(destinationBlob.OpenWrite()))
{
writer.Write(contents);
}
}
}
}
private void UpdateLatestVersionFlags(CloudBlobContainer container)
{
var blobsWithMetadata = container.ListBlobs().OfType<CloudBlockBlob>().Select(x => new
{
Blob = x,
Metadata = _packageSerializer.ReadFromMetadata(x)
}).ToList();
foreach (var blobWithMetadata in blobsWithMetadata)
blobWithMetadata.Metadata.IsLatestVersion = false;
var latest = blobsWithMetadata.OrderBy(x => x.Metadata.Version).Reverse().First();
latest.Metadata.IsLatestVersion = true;
latest.Metadata.IsAbsoluteLatestVersion = true;
foreach (var blobWithMetadata in blobsWithMetadata)
_packageSerializer.SaveToMetadata(blobWithMetadata.Metadata,blobWithMetadata.Blob);
}
public virtual void DownloadFileToBlob(BlobDownloadParameters parameters)
{
if (parameters == null || parameters.Credentials == null || string.IsNullOrWhiteSpace(parameters.SasUri.ToString()))
{
throw new ArgumentNullException(Resources.DownloadCredentialsNull);
}
CloudBlobContainer sascontainer = new CloudBlobContainer(parameters.SasUri);
var bloblist = sascontainer.ListBlobs(null, true);
string downloadFolderPath = parameters.Directory.Insert(parameters.Directory.Length, @"\");
foreach (var blob in bloblist)
{
ICloudBlob destBlob = blob as ICloudBlob;
int length = destBlob.Name.Split('/').Length;
string blobFileName = destBlob.Name.Split('/')[length-1];
// the folder structure of run logs changed from flat listing to nesting under time directory
string blobFolderPath = String.Empty;
if (destBlob.Name.Length > blobFileName.Length)
{
blobFolderPath = destBlob.Name.Substring(0, destBlob.Name.Length - blobFileName.Length - 1);
}
if (!Directory.Exists(downloadFolderPath + blobFolderPath))
{
Directory.CreateDirectory(downloadFolderPath + blobFolderPath);
}
// adding _log suffix to differentiate between files and folders of the same name. Azure blob storage only knows about blob files. We could use nested folder structure
// as part of the blob file name and thus it is possible to have a file and folder of the same name in the same location which is not acceptable for Windows file system
destBlob.DownloadToFile(downloadFolderPath + destBlob.Name + "_log", FileMode.Create);
}
}
public void CloudBlobDirectoryFlatListing()
{
foreach (String delimiter in Delimiters)
{
CloudBlobClient client = GenerateCloudBlobClient();
client.DefaultDelimiter = delimiter;
string name = GetRandomContainerName();
CloudBlobContainer container = client.GetContainerReference(name);
try
{
container.Create();
if (CloudBlobDirectorySetupWithDelimiter(container, delimiter))
{
IEnumerable <IListBlobItem> list1 = container.ListBlobs("TopDir1" + delimiter, false, BlobListingDetails.None, null, null);
List <IListBlobItem> simpleList1 = list1.ToList();
////Check if for 3 because if there were more than 3, the previous assert would have failed.
////So the only thing we need to make sure is that it is not less than 3.
Assert.IsTrue(simpleList1.Count == 3);
IListBlobItem item11 = simpleList1.ElementAt(0);
Assert.IsTrue(item11.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "Blob1"));
IListBlobItem item12 = simpleList1.ElementAt(1);
Assert.IsTrue(item12.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir1" + delimiter));
IListBlobItem item13 = simpleList1.ElementAt(2);
Assert.IsTrue(item13.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir2" + delimiter));
CloudBlobDirectory midDir2 = (CloudBlobDirectory)item13;
IEnumerable <IListBlobItem> list2 = container.ListBlobs("TopDir1" + delimiter + "MidDir1", true, BlobListingDetails.None, null, null);
List <IListBlobItem> simpleList2 = list2.ToList();
Assert.IsTrue(simpleList2.Count == 2);
IListBlobItem item21 = simpleList2.ElementAt(0);
Assert.IsTrue(item21.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir1" + delimiter + "EndDir1" + delimiter + "EndBlob1"));
IListBlobItem item22 = simpleList2.ElementAt(1);
Assert.IsTrue(item22.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir1" + delimiter + "EndDir2" + delimiter + "EndBlob2"));
IEnumerable <IListBlobItem> list3 = container.ListBlobs("TopDir1" + delimiter + "MidDir1" + delimiter, false, BlobListingDetails.None, null, null);
List <IListBlobItem> simpleList3 = list3.ToList();
Assert.IsTrue(simpleList3.Count == 2);
IListBlobItem item31 = simpleList3.ElementAt(0);
Assert.IsTrue(item31.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir1" + delimiter + "EndDir1" + delimiter));
IListBlobItem item32 = simpleList3.ElementAt(1);
Assert.IsTrue(item32.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir1" + delimiter + "EndDir2" + delimiter));
IEnumerable <IListBlobItem> list4 = midDir2.ListBlobs(true);
List <IListBlobItem> simpleList4 = list4.ToList();
Assert.IsTrue(simpleList4.Count == 2);
IListBlobItem item41 = simpleList4.ElementAt(0);
Assert.IsTrue(item41.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir2" + delimiter + "EndDir1" + delimiter + "EndBlob1"));
IListBlobItem item42 = simpleList4.ElementAt(1);
Assert.IsTrue(item42.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir2" + delimiter + "EndDir2" + delimiter + "EndBlob2"));
}
}
finally
{
container.DeleteIfExists();
}
}
}
private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime)
{
Trace.TraceInformation("Getting list of inactive packages.");
var packageIds = await ReportDataCollector.ListInactivePackageIdReports(_statisticsDatabase, reportGenerationTime);
Trace.TraceInformation("Found {0} inactive packages.", packageIds.Count);
// Collect the list of reports
var subContainer = "recentpopularity/";
Trace.TraceInformation("Collecting list of package detail reports");
var reports = destinationContainer.ListBlobs(subContainer + _recentPopularityDetailByPackageReportBaseName)
.OfType<CloudBlockBlob>()
.Select(b => b.Name);
var reportSet = new HashSet<string>(reports);
Trace.TraceInformation("Collected {0} package detail reports", reportSet.Count);
Parallel.ForEach(packageIds, new ParallelOptions { MaxDegreeOfParallelism = 8 }, async id =>
{
string reportName = _recentPopularityDetailByPackageReportBaseName + id;
string blobName = subContainer + reportName + ".json";
if (reportSet.Contains(blobName))
{
var blob = destinationContainer.GetBlockBlobReference(blobName);
Trace.TraceInformation("{0}: Deleting empty report from {1}", reportName, blob.Uri.AbsoluteUri);
await blob.DeleteIfExistsAsync();
Trace.TraceInformation("{0}: Deleted empty report from {1}", reportName, blob.Uri.AbsoluteUri);
}
});
}
