// copy a directory of the same container to another container
public static List<ICancellableAsyncResult> CopyBlobDirectory(CloudBlobDirectory srcDirectory, CloudBlobContainer destContainer, string sourceblobToken)
{
List<ICancellableAsyncResult> mylistresults = new List<ICancellableAsyncResult>();
var srcBlobList = srcDirectory.ListBlobs(
useFlatBlobListing: true,
blobListingDetails: BlobListingDetails.None).ToList();
foreach (var src in srcBlobList)
{
var srcBlob = src as ICloudBlob;
// Create appropriate destination blob type to match the source blob
ICloudBlob destBlob;
if (srcBlob.Properties.BlobType == BlobType.BlockBlob)
destBlob = destContainer.GetBlockBlobReference(srcBlob.Name);
else
destBlob = destContainer.GetPageBlobReference(srcBlob.Name);
// copy using src blob as SAS
mylistresults.Add(destBlob.BeginStartCopyFromBlob(new Uri(srcBlob.Uri.AbsoluteUri + sourceblobToken), null, null));
}
return mylistresults;
}
/// <summary>
/// Devuelve todos los archivos contenidos en un directorio de manera recursiva o no
/// </summary>
/// <param name="blobDirectory"></param>
/// <param name="isRecursive"></param>
/// <returns></returns>
public static IEnumerable<CloudBlockBlob> GetFilesFrom(CloudBlobDirectory blobDirectory, bool isRecursive)
{
foreach (var blob in blobDirectory.ListBlobs().OfType<CloudBlockBlob>())
{
yield return blob;
}
if (isRecursive == false)
yield break;
foreach (var blob in blobDirectory.ListBlobs().OfType<CloudBlobDirectory>())
{
foreach (var innerBlob in GetFilesFrom(blob, true))
{
yield return innerBlob;
}
}
}
static void DeleteBlobDirectory(CloudBlobDirectory blobDirectory)
{
foreach(var blobItem in blobDirectory.ListBlobs())
{
var subDirectory = blobItem as CloudBlobDirectory;
if (subDirectory != null)
DeleteBlobDirectory(subDirectory);
var blob = blobItem as ICloudBlob;
if (blob != null)
blob.Delete();
}
}
private static void ListBlobsOfDirectory(CloudBlobDirectory directory)
{
var listOfObjects = directory.ListBlobs();
foreach (var obj in listOfObjects)
{
var type = obj.GetType();
if (type == typeof(CloudBlockBlob))
{
Trace.TraceInformation(obj.Uri.ToString());
}
else if (type == typeof(CloudBlobDirectory))
{
ListBlobsOfDirectory((CloudBlobDirectory)obj);
}
}
}
public void CloudBlobDirectoryGetParent()
{
foreach (String delimiter in Delimiters)
{
CloudBlobClient client = GenerateCloudBlobClient();
client.DefaultDelimiter = delimiter;
string name = GetRandomContainerName();
CloudBlobContainer container = client.GetContainerReference(name);
try
{
container.Create();
CloudPageBlob blob = container.GetPageBlobReference("Dir1" + delimiter + "Blob1");
blob.Create(0);
Assert.AreEqual("Dir1" + delimiter + "Blob1", blob.Name);
// get the blob's parent
CloudBlobDirectory parent = blob.Parent;
Assert.AreEqual(parent.Prefix, "Dir1" + delimiter);
// get container as parent
CloudBlobDirectory root = parent.Parent;
Assert.AreEqual(root.Prefix, "");
// make sure the parent of the container dir is null
CloudBlobDirectory empty = root.Parent;
Assert.IsNull(empty);
// from container, get directory reference to container
root = container.GetDirectoryReference("");
Assert.AreEqual("", root.Prefix);
Assert.AreEqual(container.Uri.AbsoluteUri, root.Uri.AbsoluteUri);
List <IListBlobItem> list = root.ListBlobs().ToList();
Assert.AreEqual(1, list.Count);
// make sure the parent of the container dir is null
empty = root.Parent;
Assert.IsNull(empty);
blob.DeleteIfExists();
}
finally
{
container.DeleteIfExists();
}
}
}
private static void DeleteAllBlobsDir(CloudBlobDirectory cloudBlobDirectory)
{
foreach (var blob in cloudBlobDirectory.ListBlobs())
{
var blockBlob = blob as CloudBlockBlob;
if (blockBlob != null)
{
blockBlob.DeleteIfExists();
}
var directory = blob as CloudBlobDirectory;
if (directory != null)
{
DeleteAllBlobsDir(directory);
}
}
}
// delete folders
public void deleteFolder(CloudBlobContainer container, CloudBlobDirectory tempDir)
{
try
{
IEnumerable<IListBlobItem> blobs = tempDir.ListBlobs();
foreach (IListBlobItem item in blobs)
{
// for items in subdirecories
// parent directory doesn't need this
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob blob = (CloudBlockBlob)item;
//blob.Delete();
blob.DeleteIfExists(DeleteSnapshotsOption.IncludeSnapshots);
}
else if (item.GetType() == typeof(CloudPageBlob))
{
CloudPageBlob blob = (CloudPageBlob)item;
blob.DeleteIfExists(DeleteSnapshotsOption.IncludeSnapshots);
}
//else if (item.GetType() == typeof(CloudBlobDirectory))
//{
// CloudBlobDirectory blob = (CloudBlobDirectory)item;
// blob.
//}
else
{
// delete folders recursively
CloudBlobDirectory blobDir = (CloudBlobDirectory)item;
deleteFolder(container, blobDir);
}
}
}
catch
{
throw;
}
}
public AzureBlobRepository OperOrCreate(CloudBlobDirectory directory, AzureBlobShardingStrategy defaultShardingStrategy)
{
AzureBlobShardingStrategy sharding = defaultShardingStrategy;
var blobs = directory.ListBlobs().ToList();
var dataBlobs = blobs.Where(s => !s.Uri.Segments.Last().StartsWith(Sharding)).ToList();
if (!blobs.Any())
{
var id = GetId(sharding);
var factoryBlob = directory.GetPageBlobReference(Get_Factory_Blob(directory, id.ToString()));
factoryBlob.Create(0);
}
else
{
var factory = blobs.Select(s=> s.Uri.Segments.Last()).Single(b => b.StartsWith(Sharding));
int spearatorIndex = factory.IndexOf('-');
var id = factory.Substring(spearatorIndex + 1);
sharding = BuildShardingStrategy(id);
}
return new AzureBlobRepository(directory, sharding, _builder);
}
private static long GetDirectorySize(CloudBlobDirectory directoryBlob) {
long size = 0;
foreach (var blobItem in directoryBlob.ListBlobs()) {
if (blobItem is CloudBlockBlob)
size += ((CloudBlockBlob)blobItem).Properties.Length;
if (blobItem is CloudBlobDirectory)
size += GetDirectorySize((CloudBlobDirectory)blobItem);
}
return size;
}
public void CloudBlobDirectoryFlatListing()
{
foreach (String delimiter in Delimiters)
{
CloudBlobClient client = GenerateCloudBlobClient();
client.DefaultDelimiter = delimiter;
string name = GetRandomContainerName();
CloudBlobContainer container = client.GetContainerReference(name);
try
{
container.Create();
if (CloudBlobDirectorySetupWithDelimiter(container, delimiter))
{
IEnumerable <IListBlobItem> list1 = container.ListBlobs("TopDir1" + delimiter, false, BlobListingDetails.None, null, null);
List <IListBlobItem> simpleList1 = list1.ToList();
////Check if for 3 because if there were more than 3, the previous assert would have failed.
////So the only thing we need to make sure is that it is not less than 3.
Assert.IsTrue(simpleList1.Count == 3);
IListBlobItem item11 = simpleList1.ElementAt(0);
Assert.IsTrue(item11.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "Blob1"));
IListBlobItem item12 = simpleList1.ElementAt(1);
Assert.IsTrue(item12.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir1" + delimiter));
IListBlobItem item13 = simpleList1.ElementAt(2);
Assert.IsTrue(item13.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir2" + delimiter));
CloudBlobDirectory midDir2 = (CloudBlobDirectory)item13;
IEnumerable <IListBlobItem> list2 = container.ListBlobs("TopDir1" + delimiter + "MidDir1", true, BlobListingDetails.None, null, null);
List <IListBlobItem> simpleList2 = list2.ToList();
Assert.IsTrue(simpleList2.Count == 2);
IListBlobItem item21 = simpleList2.ElementAt(0);
Assert.IsTrue(item21.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir1" + delimiter + "EndDir1" + delimiter + "EndBlob1"));
IListBlobItem item22 = simpleList2.ElementAt(1);
Assert.IsTrue(item22.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir1" + delimiter + "EndDir2" + delimiter + "EndBlob2"));
IEnumerable <IListBlobItem> list3 = container.ListBlobs("TopDir1" + delimiter + "MidDir1" + delimiter, false, BlobListingDetails.None, null, null);
List <IListBlobItem> simpleList3 = list3.ToList();
Assert.IsTrue(simpleList3.Count == 2);
IListBlobItem item31 = simpleList3.ElementAt(0);
Assert.IsTrue(item31.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir1" + delimiter + "EndDir1" + delimiter));
IListBlobItem item32 = simpleList3.ElementAt(1);
Assert.IsTrue(item32.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir1" + delimiter + "EndDir2" + delimiter));
IEnumerable <IListBlobItem> list4 = midDir2.ListBlobs(true);
List <IListBlobItem> simpleList4 = list4.ToList();
Assert.IsTrue(simpleList4.Count == 2);
IListBlobItem item41 = simpleList4.ElementAt(0);
Assert.IsTrue(item41.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir2" + delimiter + "EndDir1" + delimiter + "EndBlob1"));
IListBlobItem item42 = simpleList4.ElementAt(1);
Assert.IsTrue(item42.Uri.Equals(container.Uri + "/TopDir1" + delimiter + "MidDir2" + delimiter + "EndDir2" + delimiter + "EndBlob2"));
}
}
finally
{
container.DeleteIfExists();
}
}
}
private void CollectJobRun(CloudBlobDirectory root)
{
var allblobs = root.ListBlobs(useFlatBlobListing: false, blobListingDetails: BlobListingDetails.None);
foreach (var blob in allblobs)
{
if (blob is CloudBlobDirectory) CollectJobRun(blob as CloudBlobDirectory);
else
{
runs.Add(root.Prefix);
break;
}
}
}
private static long GetDirectorySize(CloudBlobDirectory directoryBlob) {
long size = 0;
foreach ( var blobItem in directoryBlob.ListBlobs() ) {
var blob = blobItem as CloudBlockBlob;
if (blob != null)
size += blob.Properties.Length;
var item = blobItem as CloudBlobDirectory;
if (item != null)
size += GetDirectorySize(item);
}
return size;
}
/// <summary>
/// Deletes a directory in Azure Blob
/// </summary>
/// <param name="directory">The directory.</param>
private void DeleteDirectory(CloudBlobDirectory directory)
{
try
{
foreach (var item in directory.ListBlobs())
{
var blockBlob = item as CloudBlockBlob;
if (blockBlob != null)
{
blockBlob.DeleteIfExists();
}
else
{
var dir = item as CloudBlobDirectory;
if (dir != null)
{
this.DeleteDirectory(dir);
}
}
}
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while cleaning the cluster\n" + e);
}
}
public static IEnumerable<CloudBlobDirectory> ListSubdirectories(CloudBlobDirectory cloudBlobDirectory) {
var p = cloudBlobDirectory.Uri.AbsolutePath;
var l = p.EndsWith("/") ? cloudBlobDirectory.Uri.AbsolutePath.Length : cloudBlobDirectory.Uri.AbsolutePath.Length + 1;
return (from blob in cloudBlobDirectory.ListBlobs().Select(each => each.Uri.AbsolutePath.Substring(l))
let i = blob.IndexOf('/')
where i > -1
select blob.Substring(0, i)).Distinct().Select(cloudBlobDirectory.GetSubdirectoryReference);
}
private List<IListBlobItem> ListVirtualDirectoryBlobs(CloudBlobDirectory dir)
{
var blobList = new List<IListBlobItem>();
foreach (var blob in dir.ListBlobs())
{
if (blob.GetType() == typeof(CloudBlockBlob) || blob.GetType() == typeof(CloudPageBlob) )
{
blobList.Add(blob);
}
if (blob.GetType() == typeof(CloudBlobDirectory))
{
var subList = ListVirtualDirectoryBlobs((CloudBlobDirectory)blob);
blobList.AddRange(subList);
}
}
return blobList;
}
