void CopyDirectory(CloudBlobDirectory dir, string destinationKey, bool deleteSource = false)
{
var children = new List <IListBlobItem>();
BlobContinuationToken continuationToken = null;
do
{
BlobResultSegment segmentResult = dir.ListBlobsSegmented(continuationToken);
continuationToken = segmentResult.ContinuationToken;
children.AddRange(segmentResult.Results);
} while (continuationToken != null);
foreach (IListBlobItem blob in children)
{
string childCopyName = blob.Uri.Segments.Last();
string childCopyKey = $"{destinationKey}{childCopyName}";
if (blob is CloudBlob)
{
CopyFile((CloudBlob)blob, childCopyKey, deleteSource);
}
else if (blob is CloudBlobDirectory)
{
CopyDirectory((CloudBlobDirectory)blob, childCopyKey, deleteSource);
}
else
{
throw new Exception("Unsupported blob type");
}
}
}
private BlobResultSegment GetNextHeartbeats(CloudBlobDirectory directory, BlobContinuationToken currentToken)
{
const int batchSize = 100;
try
{
return(directory.ListBlobsSegmented(useFlatBlobListing: true,
blobListingDetails: BlobListingDetails.None,
maxResults: batchSize,
currentToken: currentToken,
options: null,
operationContext: null));
}
catch (StorageException exception)
{
if (exception.IsNotFound())
{
return(null);
}
else
{
throw;
}
}
}
void RemoveDirectory(CloudBlobDirectory dir)
{
var children = new List <IListBlobItem>();
BlobContinuationToken continuationToken = null;
do
{
BlobResultSegment segmentResult = dir.ListBlobsSegmented(continuationToken);
continuationToken = segmentResult.ContinuationToken;
children.AddRange(segmentResult.Results);
} while (continuationToken != null);
foreach (IListBlobItem blob in children)
{
if (blob is CloudBlob)
{
RemoveFile((CloudBlob)blob);
}
else if (blob is CloudBlobDirectory)
{
RemoveDirectory((CloudBlobDirectory)blob);
}
else
{
throw new Exception("Unsupported blob type");
}
}
}
private static void DownloadBlobs(string downloadedBlobsPath, string containerName, StorageAccountFactory storageAccountFactory)
{
CloudStorageAccount storageAccount = null;
try
{
storageAccount = storageAccountFactory.GetStorageAccount();
// Create the blob client object
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Get the container reference
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlobDirectory directory = container.GetDirectoryReference(string.Join("/", TestFabricNodeName, "Fabric"));
BlobContinuationToken continuationToken = null;
BlobRequestOptions requestOptions = new BlobRequestOptions();
OperationContext operationContext = new OperationContext();
var blobResultSegment = directory.ListBlobsSegmented(true, BlobListingDetails.All, null, continuationToken, requestOptions, operationContext);
int zipSequence = 1;
foreach (ICloudBlob blob in blobResultSegment.Results)
{
try
{
var localFilePath = Path.Combine(downloadedBlobsPath, string.Concat(zipSequence++, ".dtr.zip"));
blob.DownloadToFile(localFilePath, FileMode.Create);
var outputFilePath = localFilePath.Substring(0, localFilePath.Length - 4);
ZipFile.ExtractToDirectory(localFilePath, outputFilePath);
DirectoryInfo outputDir = new DirectoryInfo(outputFilePath);
foreach (var file in outputDir.GetFiles())
{
File.Copy(file.FullName, Path.Combine(downloadedBlobsPath, file.Name));
File.Delete(localFilePath);
Directory.Delete(outputDir.FullName, true);
}
}
catch (Exception ex)
{
Utility.TraceSource.WriteError(
TraceType,
"Failed to download and unzip blob: {0}",
ex);
}
}
}
finally
{
// Replace real key with a bogus key and then clear out the real key.
if (storageAccount != null)
{
storageAccount.Credentials.UpdateKey(
AzureUtility.DevelopmentStorageAccountWellKnownKey,
storageAccount.Credentials.KeyName);
}
}
}
public StorageDirectoryContents(CloudBlobDirectory blob)
{
BlobContinuationToken continuationToken = null;
do
{
var response = blob.ListBlobsSegmented(continuationToken);
continuationToken = response.ContinuationToken;
_blobs.AddRange(response.Results);
}while (continuationToken != null);
Exists = _blobs.Count > 0;
}
public IEnumerable <FileSystemItem> GetItems(FileSystemLoadItemOptions options)
{
var result = new List <FileSystemItem>();
BlobContinuationToken continuationToken = null;
string dirKey = GetFileItemPath(options.Directory);
if (!string.IsNullOrEmpty(dirKey))
{
dirKey = dirKey + "/";
}
CloudBlobDirectory dir = Container.GetDirectoryReference(dirKey);
do
{
BlobResultSegment segmentResult = dir.ListBlobsSegmented(continuationToken);
continuationToken = segmentResult.ContinuationToken;
foreach (IListBlobItem blob in segmentResult.Results)
{
var item = new FileSystemItem();
string name = GetFileItemName(blob);
if (name == EmptyDirectoryDummyBlobName)
{
continue;
}
if (blob is CloudBlob)
{
var blockBlob = (CloudBlob)blob;
item.Name = name;
item.DateModified = blockBlob.Properties.LastModified.GetValueOrDefault().DateTime;
item.Size = blockBlob.Properties.Length;
}
else if (blob is CloudBlobDirectory)
{
var subDir = (CloudBlobDirectory)blob;
item.Name = name.Substring(0, name.Length - 1);
item.IsDirectory = true;
item.HasSubDirectories = GetHasDirectories(subDir);
item.DateModified = DateTime.UtcNow;
}
else
{
throw new Exception("Unsupported blob type");
}
result.Add(item);
}
} while(continuationToken != null);
return(result.OrderByDescending(item => item.IsDirectory)
.ThenBy(item => item.Name)
.ToList());
}
bool GetHasDirectories(CloudBlobDirectory dir)
{
bool result;
BlobContinuationToken continuationToken = null;
do
{
BlobResultSegment segmentResult = dir.ListBlobsSegmented(continuationToken);
continuationToken = segmentResult.ContinuationToken;
result = segmentResult.Results.Any(blob => blob is CloudBlobDirectory);
} while (!result && continuationToken != null);
return(result);
}
public IEnumerable <IListBlobItem> ListBlobsSegmented(string containerName, string directoryPath)
{
// Retrieve a reference to a container
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlobDirectory dir = container.GetDirectoryReference(directoryPath);
if (dir == null)
{
return(null);
}
return(dir.ListBlobsSegmented(null).Results);
}
public static List <IListBlobItem> GetItems(this CloudBlobDirectory directory, int maxResults = 100)
{
var blobContinuationToken = new BlobContinuationToken();
var blobRequestOptions = new BlobRequestOptions();
var operationContext = new OperationContext();
var results = new List <IListBlobItem>();
do
{
var result = directory.ListBlobsSegmented(false, BlobListingDetails.Metadata, maxResults, blobContinuationToken, blobRequestOptions, operationContext);
blobContinuationToken = result.ContinuationToken;
results.AddRange(result.Results);
}while (blobContinuationToken != null);
return(results);
}
/// <summary>
/// A custom validation that validates that the input (if provided) represents either a blob file or a non-empty Azure blob directory
/// </summary>
public static ValidationResult ValidateBlobExistsOrANonEmptyBlobDirectory(string blobOrDirectoryRelativePath, ValidationContext context)
{
var modelParameters = (ModelParameters)context.ObjectInstance;
try
{
if (string.IsNullOrWhiteSpace(blobOrDirectoryRelativePath))
{
return(ValidationResult.Success);
}
// create the container
CloudBlobContainer container = GetBlobContainerIfExists(modelParameters.BlobContainerName);
if (container == null)
{
// the container is invalid - skip this validation
return(ValidationResult.Success);
}
blobOrDirectoryRelativePath = blobOrDirectoryRelativePath.Replace('\\', '/').Trim('/');
// check if the input path represents a single blob
if (container.GetBlobReference(blobOrDirectoryRelativePath).Exists())
{
return(ValidationResult.Success);
}
// create a reference to the blob directory
CloudBlobDirectory blobDirectory = container.GetDirectoryReference(blobOrDirectoryRelativePath);
// list the directory blobs to validate at least one block blob exists
BlobResultSegment segment =
blobDirectory.ListBlobsSegmented(false, BlobListingDetails.None, 1, null, null, null);
if (!segment.Results.Any())
{
return(new ValidationResult($"{context.DisplayName} does not contain any blobs"));
}
}
catch (Exception e)
{
// could not verify folder contents, continue anyways
Trace.TraceWarning($"Unexpected exception '{e}' - Internal API '{nameof(ValidateBlobExistsOrANonEmptyBlobDirectory)}'");
}
return(ValidationResult.Success);
}
public void ListAndHandle(CloudBlobDirectory dir,
bool flatBlobListing = false,
Action <ICloudBlob> blobAction = null,
Action <CloudBlobDirectory> dirAction = null,
Action <CloudBlobContainer> containerAction = null)
{
BlobContinuationToken token = null;
while (true)
{
var r = dir.ListBlobsSegmented(flatBlobListing, BlobListingDetails.None, 10, token, null, null);
token = r.ContinuationToken;
var blobs = r.Results;
this.HandleItems(blobs, blobAction, dirAction, containerAction, dir.Prefix);
if (token == null)
{
break;
}
}
}
public static BlobResultSegment GetBlobList(CloudBlobDirectory directory, bool useFlatBlobListing, BlobContinuationToken blobContinuationToken, int maxResults)
{
try
{
return(BackupRestoreUtility.PerformWithRetries <bool, BlobResultSegment>(
(bool flatBlobListing) =>
{
return directory.ListBlobsSegmented(flatBlobListing, BlobListingDetails.None, maxResults, blobContinuationToken, null, null);
},
useFlatBlobListing,
new RetriableOperationExceptionHandler(AzureStorageExceptionHandler),
RetryCount));
}
catch (Exception e)
{
TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to list blobs under directory {0} from Azure storage.",
directory.Prefix);
throw;
}
}
public AzureDirectoryContents(CloudBlobDirectory blob)
{
_blob = blob;
_directoryContent = blob.ListBlobsSegmented(null);
Exists = _directoryContent.Results != null && _directoryContent.Results.Any();
}
protected bool DirectoryExists(CloudBlobDirectory directory)
{
return(directory.ListBlobsSegmented(true, BlobListingDetails.None, 1, new BlobContinuationToken(), null, null).Results.Any());
}
public bool IsDirEmpty(CloudBlobDirectory dir)
{
var r = dir.ListBlobsSegmented(true, BlobListingDetails.None, 1, null, null, null);
return(r.Results.Count() == 0);
}
