private static async Task ProcessAsync()
{
CloudStorageAccount storageAccount = null;
CloudBlobContainer cloudBlobContainer = null;
string sourceFile = null;
string destinationFile = null;
// Retrieve the connection string for use with the application. The storage connection string is stored
// in an environment variable on the machine running the application called storageconnectionstring.
// If the environment variable is created after the application is launched in a console or with Visual
// Studio, the shell needs to be closed and reloaded to take the environment variable into account.
//string storageConnectionString = Environment.GetEnvironmentVariable("storageconnectionstring");
string storageConnectionString = "<REPLACE-WITH-CONNECTION-STRING-HERE>";
// Check whether the connection string can be parsed.
if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
{
try
{
// Create the CloudBlobClient that represents the Blob storage endpoint for the storage account.
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
// Create a container called 'book-name-' and append a GUID value to it to make the name unique.
cloudBlobContainer = cloudBlobClient.GetContainerReference("book-name-" + Guid.NewGuid().ToString());
await cloudBlobContainer.CreateAsync();
Console.WriteLine("Created container '{0}'", cloudBlobContainer.Name);
Console.WriteLine();
// Set the permissions so the blobs are public.
BlobContainerPermissions permissions = new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Blob
};
await cloudBlobContainer.SetPermissionsAsync(permissions);
// Create a file in your local MyDocuments folder to upload to a blob.
// string localPath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
// string localFileName = "QuickStart_" + Guid.NewGuid().ToString() + ".txt";
// sourceFile = Path.Combine(localPath, localFileName);
// Write text to the file.
// File.WriteAllText(sourceFile, "Hello, World!");
// Console.WriteLine("Temp file = {0}", sourceFile);
// Console.WriteLine("Uploading to Blob storage as blob '{0}'", localFileName);
// Console.WriteLine();
// Get a reference to the blob address, then upload the file to the blob.
// Use the value of localFileName for the blob name.
// CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(localFileName);
// await cloudBlockBlob.UploadFromFileAsync(sourceFile);
// List the blobs in the container.
Console.WriteLine("Listing blobs in container.");
BlobContinuationToken blobContinuationToken = null;
do
{
var results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken);
// Get the value of the continuation token returned by the listing call.
blobContinuationToken = results.ContinuationToken;
foreach (IListBlobItem item in results.Results)
{
Console.WriteLine(item.Uri);
}
} while (blobContinuationToken != null); // Loop while the continuation token is not null.
Console.WriteLine();
// Download the blob to a local file, using the reference created earlier.
// Append the string "_DOWNLOADED" before the .txt extension so that you can see both files in MyDocuments.
// destinationFile = sourceFile.Replace(".txt", "_DOWNLOADED.txt");
// Console.WriteLine("Downloading blob to {0}", destinationFile);
// Console.WriteLine();
// await cloudBlockBlob.DownloadToFileAsync(destinationFile, FileMode.Create);
}
catch (StorageException ex)
{
Console.WriteLine("Error returned from the service: {0}", ex.Message);
}
// finally
// {
// Console.WriteLine("Press any key to delete the sample files and example container.");
// Console.ReadLine();
// // Clean up resources. This includes the container and the two temp files.
// Console.WriteLine("Deleting the container and any blobs it contains");
// if (cloudBlobContainer != null)
// {
// await cloudBlobContainer.DeleteIfExistsAsync();
// }
// Console.WriteLine("Deleting the local source file and local downloaded files");
// Console.WriteLine();
// File.Delete(sourceFile);
// File.Delete(destinationFile);
// }
}
else
{
Console.WriteLine(
"A connection string has not been defined in the system environment variables. " +
"Add a environment variable named 'storageconnectionstring' with your storage " +
"connection string as a value.");
}
}
/// <summary>
/// Get a list of cloudblobcontainer in azure
/// </summary>
/// <param name="prefix">Container prefix</param>
/// <param name="detailsIncluded">Container listing details</param>
/// <param name="options">Blob request option</param>
/// <param name="operationContext">Operation context</param>
/// <returns>An enumerable collection of cloudblobcontainer</returns>
public ContainerResultSegment ListContainersSegmented(string prefix, ContainerListingDetails detailsIncluded, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, OperationContext operationContext)
{
return(this.BlobClient.ListContainersSegmented(prefix, detailsIncluded, maxResults, currentToken, options, operationContext));
}
public async Task ListFolder(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken)
{
CloudBlobDirectory dir = GetCloudBlobDirectory(path);
BlobContinuationToken token = null;
var batch = new List <BlobId>();
do
{
BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
false, BlobListingDetails.None, null, token, null, null, cancellationToken);
token = segment.ContinuationToken;
foreach (IListBlobItem blob in segment.Results)
{
BlobId id;
if (blob is CloudBlockBlob blockBlob)
{
id = new BlobId(blockBlob.Name, BlobItemKind.File);
}
else if (blob is CloudAppendBlob appendBlob)
{
id = new BlobId(appendBlob.Name, BlobItemKind.File);
}
else if (blob is CloudBlobDirectory dirBlob)
{
id = new BlobId(dirBlob.Prefix, BlobItemKind.Folder);
}
else
{
throw new InvalidOperationException($"unknown item type {blob.GetType()}");
}
if (options.IsMatch(id))
{
batch.Add(id);
}
}
}while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));
batch = batch.Where(options.IsMatch).ToList();
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
foreach (BlobId folderId in folderIds)
{
await ListFolder(
container,
StoragePath.Combine(path, folderId.Id),
options,
cancellationToken);
}
}
}
/// <summary>
/// List the blobs segmented in specified containers
/// </summary>
/// <param name="container">A cloudblobcontainer object</param>
/// <param name="prefix">Blob prefix</param>
/// <param name="useFlatBlobListing">Use flat blob listing(whether treat "container/" as directory)</param>
/// <param name="blobListingDetails">Blob listing details</param>
/// <param name="options">Blob request option</param>
/// <param name="operationContext">Operation context</param>
public Task <BlobResultSegment> ListBlobsSegmentedAsync(CloudBlobContainer container, string prefix, bool useFlatBlobListing, BlobListingDetails blobListingDetails, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken)
{
//BlobResultSegment is sealed without any public constructors.
throw new NotImplementedException();
}
private static async Task SearchBlobsAsync(CloudBlobContainer container)
{
if (await container.ExistsAsync())
{
WriteLine($"Enter the path to search, leave blank to search the entire '{container.Name}' container:");
WriteLineWithGreenColor(true);
WriteLine($"Ex: blobreceipts/hostId/namespace.functionName.Run");
WriteLineWithGreenColor(false);
var prefix = @ReadLine();
WriteLine($"Enter the start date by counting the number of days from today on which the search should start.");
WriteLineWithGreenColor(true);
WriteLine($"Ex: 1 is {DateTime.Now.AddDays(-1)} (yesterday) and 5 is {DateTime.Now.AddDays(-5)} (5 days ago)");
WriteLineWithGreenColor(false);
int startDate = (ToInt32(ReadLine()) * -1);
WriteLine($"Enter the end date by counting the number of days from today on which the search should end.");
WriteLineWithGreenColor(true);
WriteLine($"Enter 0 for now {DateTime.Now}");
WriteLineWithGreenColor(false);
int endDate = (ToInt32(ReadLine()) * -1);
if (endDate == 0)
{
endDate = 1; //Seems this logic didn't work when endDate was 0, but nothing can already exist which is added tomorrow...
}
if (startDate > endDate)
{
WriteLine($"Start date {DateTime.Now.AddDays(startDate)} " +
$"cannot come before end date {DateTime.Now.AddDays(endDate)}, start over.");
return;
}
WriteLineWithGreenColor(true);
WriteLine($"Searching '{container.Name} -> {prefix}' from {DateTime.Now.AddDays(startDate)} " +
$"to {DateTime.Now.AddDays(endDate)}");
WriteLineWithGreenColor(false);
try
{
int maxResults = 500;
BlobContinuationToken continuationToken = null;
CloudBlob blob;
IEnumerable <IListBlobItem> blobList;
do
{
BlobResultSegment resultSegment = await container.ListBlobsSegmentedAsync(prefix,
true, BlobListingDetails.Metadata, maxResults, continuationToken, null, null);
blobList = resultSegment.Results.OfType <CloudBlob>()
.Where(b => b.Properties.Created >= DateTime.Today.AddDays(startDate) &&
b.Properties.Created <= DateTime.Today.AddDays(endDate))
.Select(b => b);
foreach (var blobItem in blobList)
{
blob = (CloudBlob)blobItem;
await blob.FetchAttributesAsync();
WriteLine($"Blob name: {blob.Name} - last modified on {blob.Properties.LastModified}");
}
continuationToken = resultSegment.ContinuationToken;
} while (continuationToken != null);
if (blobList.Count() > 0)
{
WriteLine("Would you like to remove/reprocess a blob? Y/N ");
var delete = ReadLine();
while (delete == "Y")
{
//should repopulate blobList and check there are blobs to delete
WriteLine("Enter the path and blob name you would like to remove/reprocess: ");
WriteLineWithGreenColor(true);
WriteLine($"Ex: {((CloudBlob)blobList.First()).Name}");
WriteLineWithGreenColor(false);
var path = ReadLine();
CloudBlockBlob blockBlob = container.GetBlockBlobReference(path);
await blockBlob.DeleteIfExistsAsync();
WriteLine($"Deleted {path} from {container.Name}");
WriteLine("Would you like to remove/reprocess another blob? Y/N ");
delete = ReadLine();
}
}
}
catch (StorageException e)
{
WriteLine(e.Message);
ReadLine();
}
if (container.Name.ToLower() != "azure-webjobs-hosts")
{
WriteLineWithYellowColor(true);
WriteLine($"NOTE: you searched '{container.Name} -> {prefix}'. You need to search in the " +
"azure-webjobs-hosts container if you want to reprocess a blob.");
WriteLineWithYellowColor(false);
}
}
else
{
WriteLine($"Blob container '{container.Name}' doesn't exist. Please start over.");
}
}
/// <summary>Initializes a new instance of the <see cref="StorageBlobResultSegment"/> class.</summary>
/// <param name="continuationToken">The continuation token.</param>
/// <param name="results">The results in the segment.</param>
public StorageBlobResultSegment(BlobContinuationToken continuationToken,
IEnumerable <IStorageListBlobItem> results)
{
_continuationToken = continuationToken;
_results = results;
}
/// <summary>
/// List part of blobs.
/// </summary>
/// <param name="prefix">Blob prefix</param>
/// <param name="useFlatBlobListing">Use flat blob listing</param>
/// <param name="blobListingDetails">Blob listing details.</param>
/// <param name="maxResults">Max results.</param>
/// <param name="currentToken">Current token.</param>
/// <param name="options">Request options</param>
/// <param name="operationContext">Operation Context.</param>
/// <returns>BlobResultSegment object</returns>
public BlobResultSegment ListBlobsSegmented(CloudBlobContainer container, string prefix, bool useFlatBlobListing, BlobListingDetails blobListingDetails, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, OperationContext operationContext)
{
throw new NotImplementedException("Can not create a BlobResultSegment object");
}
/// <summary>
/// Get a list of cloudblobcontainer in azure
/// </summary>
/// <param name="prefix">Container prefix</param>
/// <param name="detailsIncluded">Container listing details</param>
/// <param name="options">Blob request option</param>
/// <param name="operationContext">Operation context</param>
/// <returns>An enumerable collection of cloudblobcontainer</returns>
public ContainerResultSegment ListContainersSegmented(string prefix, ContainerListingDetails detailsIncluded, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, XSCL.OperationContext operationContext)
{
try
{
return(this.BlobClient.ListContainersSegmentedAsync(prefix, detailsIncluded, maxResults, currentToken, options, operationContext).Result);
}
catch (AggregateException e) when(e.InnerException is XSCL.StorageException)
{
throw e.InnerException;
}
}
public virtual async Task <BlobEntrySearchResult> SearchAsync(string folderUrl, string keyword)
{
var retVal = AbstractTypeFactory <BlobEntrySearchResult> .TryCreateInstance();
if (!string.IsNullOrEmpty(folderUrl))
{
var blobContainer = GetBlobContainer(GetContainerNameFromUrl(folderUrl));
if (blobContainer != null)
{
BlobContinuationToken blobContinuationToken = null;
var directoryPath = GetDirectoryPathFromUrl(folderUrl);
var blobDirectory = !string.IsNullOrEmpty(directoryPath)
? blobContainer.GetDirectoryReference(directoryPath)
: null;
var listBlobs = blobDirectory != null
? await blobDirectory.ListBlobsSegmentedAsync(blobContinuationToken)
: await blobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken);
if (!string.IsNullOrEmpty(keyword))
{
if (blobDirectory != null)
{
keyword = blobDirectory.Prefix + keyword;
}
BlobContinuationToken prefixBlobContinuationToken = null;
//Only whole container list allow search by prefix
listBlobs = await blobContainer.ListBlobsSegmentedAsync(keyword, prefixBlobContinuationToken);
}
// Loop over items within the container and output the length and URI.
foreach (IListBlobItem item in listBlobs.Results)
{
if (item is CloudBlockBlob block)
{
var blobInfo = ConvertBlobToBlobInfo(block);
//Do not return empty blob (created with directory because azure blob not support direct directory creation)
if (!string.IsNullOrEmpty(blobInfo.Name))
{
retVal.Results.Add(blobInfo);
}
}
if (item is CloudBlobDirectory directory)
{
var folder = AbstractTypeFactory <BlobFolder> .TryCreateInstance();
folder.Name = Uri.UnescapeDataString(directory.Uri.AbsolutePath)
.Split(new[] { _cloudBlobClient.DefaultDelimiter },
StringSplitOptions.RemoveEmptyEntries).Last();
folder.Url = Uri.EscapeUriString(directory.Uri.ToString());
folder.ParentUrl = directory.Parent != null
? Uri.EscapeUriString(directory.Parent.Uri.ToString())
: null;
folder.RelativeUrl = folder.Url.Replace(_cloudBlobClient.BaseUri.ToString(), string.Empty);
retVal.Results.Add(folder);
}
}
}
}
else
{
BlobContinuationToken listbContinuationToken = null;
do
{
var results = await _cloudBlobClient.ListContainersSegmentedAsync(null, listbContinuationToken);
listbContinuationToken = results.ContinuationToken;
foreach (var item in results.Results)
{
var folder = AbstractTypeFactory <BlobFolder> .TryCreateInstance();
folder.Name = item.Uri.AbsolutePath.Split('/').Last();
folder.Url = Uri.EscapeUriString(item.Uri.ToString());
retVal.Results.Add(folder);
}
} while (listbContinuationToken != null);
}
retVal.TotalCount = retVal.Results.Count();
return(retVal);
}
/// <summary>
/// List the blobs segmented in specified containers
/// </summary>
/// <param name="container">A cloudblobcontainer object</param>
/// <param name="prefix">Blob prefix</param>
/// <param name="useFlatBlobListing">Use flat blob listing(whether treat "container/" as directory)</param>
/// <param name="blobListingDetails">Blob listing details</param>
/// <param name="options">Blob request option</param>
/// <param name="operationContext">Operation context</param>
public Task <BlobResultSegment> ListBlobsSegmentedAsync(CloudBlobContainer container, string prefix, bool useFlatBlobListing, BlobListingDetails blobListingDetails, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, XSCL.OperationContext operationContext, CancellationToken cancellationToken)
{
return(container.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails, maxResults, currentToken, options, operationContext, cancellationToken));
}
/// List part of blobs.
/// </summary>
/// <param name="prefix">Blob prefix</param>
/// <param name="useFlatBlobListing">Use flat blob listing</param>
/// <param name="blobListingDetails">Blob listing details.</param>
/// <param name="maxResults">Max results.</param>
/// <param name="currentToken">Current token.</param>
/// <param name="options">Request options</param>
/// <param name="operationContext">Operation Context.</param>
/// <returns>BlobResultSegment object</returns>
public BlobResultSegment ListBlobsSegmented(CloudBlobContainer container, string prefix, bool useFlatBlobListing,
BlobListingDetails blobListingDetails, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, XSCL.OperationContext operationContext)
{
try
{
return(container.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails, maxResults, currentToken, options, operationContext).Result);
}
catch (AggregateException e) when(e.InnerException is XSCL.StorageException)
{
throw e.InnerException;
}
}
public static async Task <AssetEntry> GenerateAssetInformation(ConfigWrapper config,
IAzureMediaServicesClient client, Asset asset, VodSemaphore semaphore, string contentId = null, string region = null)
{
var assetEntry = new AssetEntry()
{
AMSAccountName = config.AccountName,
Region = config.Region,
ResourceGroup = config.ResourceGroup,
AssetStorageAccountName = asset?.StorageAccountName,
AssetName = asset.Name,
Urn = semaphore?.Urn,
Semaphore = semaphore,
StreamingLocators = new List <StreamingLocatorEntry>(),
CreatedTime = asset.Created.ToUniversalTime().ToString(AssetEntry.DateFormat),
ContentId = contentId,
};
var urls = new List <OutputUrl>();
string streamingPolicyName = null;
StreamingPolicy streamingPolicy = null;
var streamingLocatorsNames = client.Assets.ListStreamingLocators(config.ResourceGroup, config.AccountName, asset.Name).StreamingLocators.Select(l => l.Name);
foreach (var locatorName in streamingLocatorsNames)
{
string cenckeyId = null;
string cbcskeyId = null;
StreamingLocator streamingLocator = null;
if (locatorName != null)
{
streamingLocator = client.StreamingLocators.Get(config.ResourceGroup,
config.AccountName, locatorName);
if (streamingLocator != null)
{
streamingPolicyName = streamingLocator.StreamingPolicyName;
if (streamingLocator.ContentKeys
.Where(k => k.LabelReferenceInStreamingPolicy == IrdetoHelpers.labelCenc)
.FirstOrDefault() != null && streamingLocator.ContentKeys
.Where(k => k.LabelReferenceInStreamingPolicy == IrdetoHelpers.labelCbcs)
.FirstOrDefault() != null)
{
cenckeyId = streamingLocator.ContentKeys
.Where(k => k.LabelReferenceInStreamingPolicy == IrdetoHelpers.labelCenc)
.FirstOrDefault().Id.ToString();
cbcskeyId = streamingLocator.ContentKeys
.Where(k => k.LabelReferenceInStreamingPolicy == IrdetoHelpers.labelCbcs)
.FirstOrDefault().Id.ToString();
}
// let's get the manifest name
string manifestName = null;
List <IListBlobItem> blobs = new List <IListBlobItem>();
try
{
ListContainerSasInput input = new ListContainerSasInput()
{
Permissions = AssetContainerPermission.Read,
ExpiryTime = DateTime.Now.AddHours(2).ToUniversalTime()
};
var responseListSas = client.Assets.ListContainerSas(config.ResourceGroup, config.AccountName, asset.Name, input.Permissions, input.ExpiryTime);
string uploadSasUrl = responseListSas.AssetContainerSasUrls.First();
var sasUri = new Uri(uploadSasUrl);
var container = new CloudBlobContainer(sasUri);
BlobContinuationToken continuationToken = null;
do
{
var response = await container.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
blobs.AddRange(response.Results);
}while (continuationToken != null);
// let's take the first manifest file. It should exist
manifestName = blobs.Where(b => (b.GetType() == typeof(CloudBlockBlob))).Select(b => (CloudBlockBlob)b).Where(b => b.Name.ToLower().EndsWith(".ism")).FirstOrDefault().Name;
}
catch
{
}
if (manifestName != null) // there is a manifest
{
urls = MediaServicesHelpers.GetUrls(config, client, streamingLocator, manifestName, true, true, true, true, true);
}
else // no manifest
{
urls = MediaServicesHelpers.GetDownloadUrls(config, client, streamingLocator, blobs);
}
}
}
if (streamingPolicyName != null)
{
streamingPolicy = client.StreamingPolicies.Get(config.ResourceGroup, config.AccountName,
streamingPolicyName);
}
var drmlist = new List <Drm>();
if (streamingPolicy != null)
{
if (streamingPolicy.CommonEncryptionCbcs != null)
{
var enProt =
MediaServicesHelpers.ReturnOutputProtocolsListCbcs(streamingPolicy
.CommonEncryptionCbcs.EnabledProtocols);
drmlist.Add(new Drm
{
Type = "FairPlay",
LicenseUrl =
streamingPolicy?.CommonEncryptionCbcs?.Drm.FairPlay
.CustomLicenseAcquisitionUrlTemplate.Replace("{ContentKeyId}", cbcskeyId).Replace("{AlternativeMediaId}", streamingLocator.AlternativeMediaId),
Protocols = enProt,
CertificateUrl = config.IrdetoFairPlayCertificateUrl
});
}
if (streamingPolicy.CommonEncryptionCenc != null)
{
var enProtW =
MediaServicesHelpers.ReturnOutputProtocolsListCencWidevine(streamingPolicy
.CommonEncryptionCbcs.EnabledProtocols);
var enProtP =
MediaServicesHelpers.ReturnOutputProtocolsListCencPlayReady(streamingPolicy
.CommonEncryptionCbcs.EnabledProtocols);
drmlist.Add(new Drm
{
Type = "PlayReady",
LicenseUrl = streamingPolicy?.CommonEncryptionCenc?.Drm.PlayReady
.CustomLicenseAcquisitionUrlTemplate.Replace("{AlternativeMediaId}", streamingLocator.AlternativeMediaId),
Protocols = enProtP
});
drmlist.Add(new Drm
{
Type = "Widevine",
LicenseUrl = streamingPolicy?.CommonEncryptionCenc?.Drm.Widevine
.CustomLicenseAcquisitionUrlTemplate.Replace("{AlternativeMediaId}", streamingLocator.AlternativeMediaId),
Protocols = enProtW
});
}
}
var StreamingLocatorInfo = new StreamingLocatorEntry
{
StreamingLocatorName = locatorName,
StreamingPolicyName = streamingPolicyName,
CencKeyId = cenckeyId,
CbcsKeyId = cbcskeyId,
Drm = drmlist,
Urls = urls.Select(url => new UrlEntry {
Protocol = url.Protocol.ToString(), Url = url.Url
})
.ToList()
};
assetEntry.StreamingLocators.Add(StreamingLocatorInfo);
}
return(assetEntry);
}
/// <summary>
/// Async get container permission
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="taskId">Task id</param>
/// <param name="context">Azure storage context</param>
/// <returns></returns>
internal async Task GetContainerPermission(long taskId, IStorageBlobManagement localChannel, CloudBlobContainer container, BlobContinuationToken continuationToken)
{
BlobRequestOptions requestOptions = RequestOptions;
AccessCondition accessCondition = null;
BlobContainerPermissions permissions = null;
try
{
permissions = await localChannel.GetContainerPermissionsAsync(container, accessCondition,
requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false);
}
catch (StorageException e) when(e.IsNotFoundException() || e.IsForbiddenException())
{
// 404 Not found, or 403 Forbidden means we don't have permission to query the Permission of the specified container.
// Just skip return container permission in this case.
}
WriteCloudContainerObject(taskId, localChannel, container, permissions, continuationToken);
}
private static async Task ProcessAsync()
{
// Copy the connection string from the portal in the variable below.
string storageConnectionString = "DefaultEndpointsProtocol=https;AccountName=20201205cshsuzy;AccountKey=HpKW/kvfuTmns8hX5BB3SQpb/w2AbeZyIIGFM6C3Tesjc+Nl6wSouPpd24fYpz1dwgOzSbKRFuXi3ONq+30k9A==;EndpointSuffix=core.windows.net";
// Check whether the connection string can be parsed.
CloudStorageAccount storageAccount;
if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
{
// Run the example code if the connection string is valid.
Console.WriteLine("Valid connection string.\r\n");
// EXAMPLE CODE STARTS BELOW HERE
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
// Create a container called 'quickstartblobs' and
// append a GUID value to it to make the name unique.
CloudBlobContainer cloudBlobContainer =
cloudBlobClient.GetContainerReference("demoblobs" +
Guid.NewGuid().ToString());
await cloudBlobContainer.CreateAsync();
Console.WriteLine("A container has been created, note the " +
"'Public access level' in the portal.");
Console.WriteLine("Press 'Enter' to continue.");
Console.ReadLine();
// Create a file in your local MyDocuments folder to upload to a blob.
string localPath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string localFileName = "BlobDemo_" + Guid.NewGuid().ToString() + ".txt";
string sourceFile = Path.Combine(localPath, localFileName);
// Write text to the file.
File.WriteAllText(sourceFile, "Hello, World!");
Console.WriteLine("\r\nTemp file = {0}", sourceFile);
Console.WriteLine("Uploading to Blob storage as blob '{0}'", localFileName);
// Get a reference to the blob address, then upload the file to the blob.
// Use the value of localFileName for the blob name.
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(localFileName);
await cloudBlockBlob.UploadFromFileAsync(sourceFile);
Console.WriteLine("\r\nVerify the creation of the blob and upload in the portal.");
Console.WriteLine("Press 'Enter' to continue.");
Console.ReadLine();
// List the blobs in the container.
Console.WriteLine("List blobs in container.");
BlobContinuationToken blobContinuationToken = null;
do
{
var results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken);
// Get the value of the continuation token returned by the listing call.
blobContinuationToken = results.ContinuationToken;
foreach (IListBlobItem item in results.Results)
{
Console.WriteLine(item.Uri);
}
} while (blobContinuationToken != null); // Loop while the continuation token is not null.
Console.WriteLine("\r\nCompare the list in the console to the portal.");
Console.WriteLine("Press 'Enter' to continue.");
Console.ReadLine();
//create SAS
SharedAccessBlobPolicy sasConstraints = new SharedAccessBlobPolicy();
sasConstraints.SharedAccessStartTime = DateTimeOffset.UtcNow.AddMinutes(-5);
sasConstraints.SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1);
sasConstraints.Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write;
var sasBlobToken = cloudBlockBlob.GetSharedAccessSignature(sasConstraints);
Console.WriteLine($"\r\nSAS url: {cloudBlockBlob.Uri + sasBlobToken}");
Console.WriteLine("Press 'Enter' to continue.");
Console.ReadLine();
// Download the blob to a local file, using the reference created earlier.
// Append the string "_DOWNLOADED" before the .txt extension so that you
// can see both files in MyDocuments.
string destinationFile = sourceFile.Replace(".txt", "_DOWNLOADED.txt");
Console.WriteLine("Downloading blob to {0}", destinationFile);
await cloudBlockBlob.DownloadToFileAsync(destinationFile, FileMode.Create);
Console.WriteLine("\r\nLocate the local file to verify it was downloaded.");
Console.WriteLine("Press 'Enter' to continue.");
Console.ReadLine();
// Clean up the resources created by the app
Console.WriteLine("Press the 'Enter' key to delete the example files " +
"and example container.");
Console.ReadLine();
// Clean up resources. This includes the container and the two temp files.
Console.WriteLine("Deleting the container");
if (cloudBlobContainer != null)
{
await cloudBlobContainer.DeleteIfExistsAsync();
}
Console.WriteLine("Deleting the source, and downloaded files\r\n");
File.Delete(sourceFile);
File.Delete(destinationFile);
}
else
{
// Otherwise, let the user know that they need to define the environment variable.
Console.WriteLine(
"A valid connection string has not been defined in the storageConnectionString variable.");
Console.WriteLine("Press enter to exit the application.");
Console.ReadLine();
}
}
/// <summary>
/// Get activities for a conversation (Aka the transcript)
/// </summary>
/// <param name="channelId">Channel Id.</param>
/// <param name="conversationId">Conversation Id.</param>
/// <param name="continuationToken">Continuatuation token to page through results.</param>
/// <param name="startDate">Earliest time to include.</param>
/// <returns></returns>
public async Task <PagedResult <IActivity> > GetTranscriptActivities(string channelId, string conversationId, string continuationToken = null, DateTime startDate = default(DateTime))
{
if (String.IsNullOrEmpty(channelId))
{
throw new ArgumentNullException($"missing {nameof(channelId)}");
}
if (String.IsNullOrEmpty(conversationId))
{
throw new ArgumentNullException($"missing {nameof(conversationId)}");
}
var pagedResult = new PagedResult <IActivity>();
var dirName = GetDirName(channelId, conversationId);
var dir = this.Container.Value.GetDirectoryReference(dirName);
int pageSize = 20;
BlobContinuationToken token = null;
List <CloudBlockBlob> blobs = new List <CloudBlockBlob>();
do
{
var segment = await dir.ListBlobsSegmentedAsync(false, BlobListingDetails.Metadata, null, token, null, null);
foreach (var blob in segment.Results.Cast <CloudBlockBlob>())
{
if (DateTime.Parse(blob.Metadata["Timestamp"]).ToUniversalTime() >= startDate)
{
if (continuationToken != null)
{
if (blob.Name == continuationToken)
{
// we found continuation token
continuationToken = null;
}
// skip record
}
else
{
blobs.Add(blob);
if (blobs.Count == pageSize)
{
break;
}
}
}
}
if (segment.ContinuationToken != null)
{
token = segment.ContinuationToken;
}
} while (token != null && blobs.Count < pageSize);
pagedResult.Items = blobs
.Select(async bl =>
{
var json = await bl.DownloadTextAsync();
return(JsonConvert.DeserializeObject <Activity>(json));
})
.Select(t => t.Result)
.ToArray();
if (pagedResult.Items.Length == pageSize)
{
pagedResult.ContinuationToken = blobs.Last().Name;
}
return(pagedResult);
}
/// <summary>
/// Basic operations to work with page blobs
/// </summary>
/// <returns>Task</returns>
private static async Task BasicStoragePageBlobOperationsAsync()
{
const string PageBlobName = "samplepageblob";
// Retrieve storage account information from connection string
// How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx
CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create a blob client for interacting with the blob service.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Create a container for organizing blobs within the storage account.
Console.WriteLine("1. Creating Container");
CloudBlobContainer container = blobClient.GetContainerReference("democontainerpageblob");
await container.CreateIfNotExistsAsync();
// Create a page blob in the newly created container.
Console.WriteLine("2. Creating Page Blob");
CloudPageBlob pageBlob = container.GetPageBlobReference(PageBlobName);
await pageBlob.CreateAsync(512 * 2 /*size*/); // size needs to be multiple of 512 bytes
// Write to a page blob
Console.WriteLine("2. Write to a Page Blob");
byte[] samplePagedata = new byte[512];
Random random = new Random();
random.NextBytes(samplePagedata);
await pageBlob.UploadFromByteArrayAsync(samplePagedata, 0, samplePagedata.Length);
// List all blobs in this container. Because a container can contain a large number of blobs the results
// are returned in segments (pages) with a maximum of 5000 blobs per segment. You can define a smaller size
// using the maxResults parameter on ListBlobsSegmentedAsync.
Console.WriteLine("3. List Blobs in Container");
BlobContinuationToken token = null;
do
{
BlobResultSegment resultSegment = await container.ListBlobsSegmentedAsync(token);
token = resultSegment.ContinuationToken;
foreach (IListBlobItem blob in resultSegment.Results)
{
// Blob type will be CloudBlockBlob, CloudPageBlob or CloudBlobDirectory
Console.WriteLine("{0} (type: {1}", blob.Uri, blob.GetType());
}
} while (token != null);
// Read from a page blob
//Console.WriteLine("4. Read from a Page Blob");
int bytesRead = await pageBlob.DownloadRangeToByteArrayAsync(samplePagedata, 0, 0, samplePagedata.Count());
// Clean up after the demo
Console.WriteLine("5. Delete page Blob");
await pageBlob.DeleteAsync();
// When you delete a container it could take several seconds before you can recreate a container with the same
// name - hence to enable you to run the demo in quick succession the container is not deleted. If you want
// to delete the container uncomment the line of code below.
//Console.WriteLine("6. Delete Container");
//await container.DeleteAsync();
}
public async Task <IAzureBlobResultSegment> ListBlobsSegmentedAsync(BlobListing blobListing, BlobListingDetails blobListingDetails, int?maxResults, BlobContinuationToken currentToken)
{
return(new HostedAzureBlobResultSegment(await _cloudBlobDirectory.ListBlobsSegmentedAsync(
blobListing == BlobListing.Flat,
blobListingDetails,
maxResults,
currentToken,
null,
null)));
}
public async Task <bool> ExecuteAsync()
{
try
{
Log.LogMessage("Performing blob merge...");
if (string.IsNullOrEmpty(SourceBlobDirectory) || string.IsNullOrEmpty(TargetBlobDirectory))
{
Log.LogError($"Please specify a source blob directory and a target blob directory");
}
else
{
// Canonicalize the target uri
string targetUri = GetCanonicalStorageUri(TargetBlobDirectory);
// Invoke the blob URI parser on the target URI and deal with any container creation that needs to happen
BlobUrlInfo targetUrlInfo = new BlobUrlInfo(targetUri);
CloudStorageAccount storageAccount = new CloudStorageAccount(new WindowsAzure.Storage.Auth.StorageCredentials(targetUrlInfo.AccountName, AccountKey), true);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
CloudBlobContainer targetContainer = client.GetContainerReference(targetUrlInfo.ContainerName);
if (!SkipCreateContainer)
{
Log.LogMessage($"Creating container {targetUrlInfo.ContainerName} if it doesn't exist.");
await targetContainer.CreateIfNotExistsAsync();
}
string sourceUri = GetCanonicalStorageUri(SourceBlobDirectory);
// Grab the source blob path from the source info and combine with the target blob path.
BlobUrlInfo sourceBlobInfo = new BlobUrlInfo(sourceUri);
// For now the source and target storage accounts should be the same, so the same account key is used for each.
if (sourceBlobInfo.AccountName != targetUrlInfo.AccountName)
{
Log.LogError($"Source and target storage accounts should be identical");
}
else
{
CloudBlobContainer sourceContainer = client.GetContainerReference(sourceBlobInfo.ContainerName);
Log.LogMessage($"Listing blobs in {sourceUri}");
// Get all source URI's with the blob prefix
BlobContinuationToken token = null;
List <IListBlobItem> sourceBlobs = new List <IListBlobItem>();
do
{
BlobResultSegment segment = await sourceContainer.ListBlobsSegmentedAsync(sourceBlobInfo.BlobPath, true,
BlobListingDetails.None, null, token, new BlobRequestOptions(), null);
token = segment.ContinuationToken;
sourceBlobs.AddRange(segment.Results);
}while (token != null);
// Ensure the source exists
if (!SkipIfMissing && sourceBlobs.Count == 0)
{
Log.LogError($"No blobs found in {sourceUri}");
}
await Task.WhenAll(sourceBlobs.Select(async blob =>
{
// Determine the relative URI for the target. This works properly when the
// trailing slash is left off of the source and target URIs.
string relativeBlobPath = blob.Uri.ToString().Substring(sourceUri.Length);
string specificTargetUri = GetCanonicalStorageUri(targetUri, relativeBlobPath);
BlobUrlInfo specificTargetBlobUrlInfo = new BlobUrlInfo(specificTargetUri);
CloudBlob targetBlob = targetContainer.GetBlobReference(specificTargetBlobUrlInfo.BlobPath);
Log.LogMessage($"Merging {blob.Uri.ToString()} into {targetBlob.Uri.ToString()}");
if (!Overwrite && await targetBlob.ExistsAsync())
{
Log.LogError($"Target blob {targetBlob.Uri.ToString()} already exists.");
}
await targetBlob.StartCopyAsync(blob.Uri);
}));
}
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
/// <summary>
/// Basic operations to work with block blobs
/// </summary>
/// <returns>A Task object.</returns>
private static async Task BasicStorageBlockBlobOperationsWithAccountSASAsync()
{
const string ImageToUpload = "HelloWorld.png";
string containerName = ContainerPrefix + Guid.NewGuid();
// Get an account SAS token.
string sasToken = GetAccountSASToken();
// Use the account SAS token to create authentication credentials.
StorageCredentials accountSAS = new StorageCredentials(sasToken);
// Informational: Print the Account SAS Signature and Token.
Console.WriteLine();
Console.WriteLine("Account SAS Signature: " + accountSAS.SASSignature);
Console.WriteLine("Account SAS Token: " + accountSAS.SASToken);
Console.WriteLine();
// Get the URI for the container.
Uri containerUri = GetContainerUri(containerName);
// Get a reference to a container using the URI and the SAS token.
CloudBlobContainer container = new CloudBlobContainer(containerUri, accountSAS);
try
{
// Create a container for organizing blobs within the storage account.
Console.WriteLine("1. Creating Container using Account SAS");
await container.CreateIfNotExistsAsync();
}
catch (StorageException e)
{
Console.WriteLine(e.ToString());
Console.WriteLine("If you are running with the default configuration, please make sure you have started the storage emulator. Press the Windows key and type Azure Storage to select and run it from the list of applications - then restart the sample.");
Console.ReadLine();
throw;
}
try
{
// To view the uploaded blob in a browser, you have two options. The first option is to use a Shared Access Signature (SAS) token to delegate
// access to the resource. See the documentation links at the top for more information on SAS. The second approach is to set permissions
// to allow public access to blobs in this container. Uncomment the line below to use this approach. Then you can view the image
// using: https://[InsertYourStorageAccountNameHere].blob.core.windows.net/democontainer/HelloWorld.png
// await container.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob });
// Upload a BlockBlob to the newly created container
Console.WriteLine("2. Uploading BlockBlob");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(ImageToUpload);
await blockBlob.UploadFromFileAsync(ImageToUpload);
// List all the blobs in the container
Console.WriteLine("3. List Blobs in Container");
BlobContinuationToken token = null;
do
{
BlobResultSegment resultSegment = await container.ListBlobsSegmentedAsync(token);
token = resultSegment.ContinuationToken;
foreach (IListBlobItem blob in resultSegment.Results)
{
// Blob type will be CloudBlockBlob, CloudPageBlob or CloudBlobDirectory
Console.WriteLine("{0} (type: {1}", blob.Uri, blob.GetType());
}
}while (token != null);
// Download a blob to your file system
Console.WriteLine("4. Download Blob from {0}", blockBlob.Uri.AbsoluteUri);
await blockBlob.DownloadToFileAsync(string.Format("./CopyOf{0}", ImageToUpload), FileMode.Create);
// Create a read-only snapshot of the blob
Console.WriteLine("5. Create a read-only snapshot of the blob");
CloudBlockBlob blockBlobSnapshot = await blockBlob.CreateSnapshotAsync(null, null, null, null);
// Delete the blob and its snapshots.
Console.WriteLine("6. Delete block Blob and all of its snapshots");
await blockBlob.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, null, null);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
Console.ReadLine();
throw;
}
finally
{
// Clean up after the demo.
// Note that it is not necessary to delete all of the blobs in the container first; they will be deleted
// with the container.
Console.WriteLine("7. Delete Container");
await container.DeleteIfExistsAsync();
}
}
/// <summary>
/// List conversations in the channelId.
/// </summary>
/// <param name="channelId">Channel Id.</param>
/// <param name="continuationToken">Continuatuation token to page through results.</param>
/// <returns>A <see cref="Task"/> A task that represents the work queued to execute.</returns>
public async Task <PagedResult <TranscriptInfo> > ListTranscriptsAsync(string channelId, string continuationToken = null)
{
if (string.IsNullOrEmpty(channelId))
{
throw new ArgumentNullException($"missing {nameof(channelId)}");
}
var dirName = GetDirName(channelId);
var dir = this.Container.Value.GetDirectoryReference(dirName);
var pageSize = 20;
BlobContinuationToken token = null;
List <TranscriptInfo> conversations = new List <TranscriptInfo>();
do
{
var segment = await dir.ListBlobsSegmentedAsync(false, BlobListingDetails.Metadata, null, token, null, null).ConfigureAwait(false);
foreach (var blob in segment.Results.Where(c => c is CloudBlobDirectory).Cast <CloudBlobDirectory>())
{
// Unescape the Id we escaped when we saved it
var conversation = new TranscriptInfo()
{
Id = Uri.UnescapeDataString(blob.Prefix.Split('/').Where(s => s.Length > 0).Last()), ChannelId = channelId
};
if (continuationToken != null)
{
if (conversation.Id == continuationToken)
{
// we found continuation token
continuationToken = null;
}
// skip record
}
else
{
conversations.Add(conversation);
if (conversations.Count == pageSize)
{
break;
}
}
}
if (segment.ContinuationToken != null)
{
token = segment.ContinuationToken;
}
}while (token != null && conversations.Count < pageSize);
var pagedResult = new PagedResult <TranscriptInfo>();
pagedResult.Items = conversations.ToArray();
if (pagedResult.Items.Length == 20)
{
pagedResult.ContinuationToken = pagedResult.Items.Last().Id;
}
return(pagedResult);
}
/// <summary>
/// Get a list of cloudblobcontainer in azure
/// </summary>
/// <param name="prefix">Container prefix</param>
/// <param name="detailsIncluded">Container listing details</param>
/// <param name="options">Blob request option</param>
/// <param name="operationContext">Operation context</param>
/// <returns>An enumerable collection of cloudblobcontainer</returns>
public ContainerResultSegment ListContainersSegmented(string prefix, ContainerListingDetails detailsIncluded, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, OperationContext operationContext)
{
throw new NotImplementedException("Can not create a ContainerResultSegment object");
}
private static async Task ProcessAsync()
{
// Retrieve the connection string for use with the application. The storage
// connection string is stored in an environment variable on the machine
// running the application called CONNECT_STR. If the
// environment variable is created after the application is launched in a
// console or with Visual Studio, the shell or application needs to be closed
// and reloaded to take the environment variable into account.
string storageConnectionString = Environment.GetEnvironmentVariable("CONNECT_STR");
//Check whether the connection string can be parsed.
CloudStorageAccount storageAccount;
if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
{
// If the connection string is valid, proceed with operations against Blob
// storage here.
// ADD OTHER OPERATIONS HERE
//Create the CloudBlobClient that represents the Blob sotrage endpoint for the storage account.
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
//create a containt called 'testingfcv' and append a GUID value to it to make the name uniqued.
CloudBlobContainer cloubBlobContainer = cloudBlobClient.GetContainerReference("testfcv" + Guid.NewGuid().ToString());
await cloubBlobContainer.CreateAsync();
//Set the permission so the blobs are public.
BlobContainerPermissions permissions = new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Blob
};
await cloubBlobContainer.SetPermissionsAsync(permissions);
//Create a file in your local MyDocuments folder to upload to a blob.
string localPath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string localFileName = "QuickStart_" + Guid.NewGuid().ToString() + ".txt";
string sourceFile = Path.Combine(localPath, localFileName);
//Write text to the file.
File.WriteAllText(sourceFile, "Hello, World!");
Console.WriteLine("Temp file = {0}", sourceFile);
Console.WriteLine("Uploading to Blob storage as blob '{0}'", localFileName);
// Get a reference to the blob address, then upload the file to the blob.
// Use the value of localFileName fore the blob name.
CloudBlockBlob cloudBlockBlob = cloubBlobContainer.GetBlockBlobReference(localFileName);
await cloudBlockBlob.UploadFromFileAsync(sourceFile);
//List the blobs in the container.
Console.WriteLine("List blobs in container.");
BlobContinuationToken blobContinuationToken = null;
do
{
var results = await cloubBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken);
//Get the value of the continuation token returned by thelisting call.
blobContinuationToken = results.ContinuationToken;
foreach (IListBlobItem item in results.Results)
{
Console.WriteLine(item.Uri);
}
} while (blobContinuationToken != null); //Loop while the continuation token is not null.
// Download the blob to a local file, using the reference created earlier.
// Append the string "_DOWNLOADED" before the .txt extension so that you
// can see both files in MyDocuments.
string destinationFile = sourceFile.Replace(".txt", "_DOWNLOADED.txt");
Console.WriteLine("Downloading blob to {0}", destinationFile);
await cloudBlockBlob.DownloadToFileAsync(destinationFile, FileMode.Create);
Console.WriteLine("Press the 'Enter' key to delete the example files, " + "example containter, and exit the application.");
Console.ReadLine();
//Clean up the resources. This includes the containter and the two temp files.
Console.WriteLine("Deleting the container");
if (cloubBlobContainer != null)
{
await cloubBlobContainer.DeleteIfExistsAsync();
}
Console.WriteLine("Deleting the source, and downloaded files");
File.Delete(sourceFile);
File.Delete(destinationFile);
}
else
{
//Otherwise, let the user know that they need to define the environment variable. "
Console.WriteLine("A connectin string has has not been defined in the system environment variables. " + "Add an environment variable named 'CONNECT_STR' with your storage " + "connection string as a value.");
Console.WriteLine("Press any key to exit the application.");
Console.ReadLine();
}
}
private static async Task ProcessAsync()
{
CloudStorageAccount storageAccount = null;
CloudBlobContainer cloudBlobContainer = null;
string sourceFile = null;
string destinationFile = null;
string storageConnectionString = Environment.GetEnvironmentVariable("storageconnectionstring");
// Check whether the connection string can be parsed.
if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
{
try
{
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
cloudBlobContainer = cloudBlobClient.GetContainerReference("quickstartblobs" + Guid.NewGuid().ToString());
await cloudBlobContainer.CreateAsync();
BlobContainerPermissions permissions = new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Blob
};
//We giv the container a public access.
await cloudBlobContainer.SetPermissionsAsync(permissions);
string localPath = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string localFileName = "QuickStart_" + Guid.NewGuid().ToString() + ".txt";
// Saving the file path.
sourceFile = Path.Combine(localPath, localFileName);
// We write into a new text file.
File.WriteAllText(sourceFile, "Hello, Acom Team!");
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(sourceFile);
await cloudBlockBlob.UploadFromFileAsync(sourceFile);
BlobContinuationToken blobContinuationToken = null;
do
{
var results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken);
blobContinuationToken = results.ContinuationToken;
foreach (IListBlobItem item in results.Results)
{
Console.WriteLine(item.Uri);
}
} while (blobContinuationToken != null);
Console.WriteLine();
destinationFile = sourceFile.Replace(".txt", "_DOWNLOADED.txt");
await cloudBlockBlob.DownloadToFileAsync(destinationFile, FileMode.Create);
}
catch (StorageException ex)
{
Console.WriteLine("Error returned from the service: {0}", ex.Message);
}
finally
{
Console.WriteLine("Press any key to delete the sample files and example container.");
Console.ReadKey();
if (cloudBlobContainer != null)
{
await cloudBlobContainer.DeleteIfExistsAsync();
}
Console.WriteLine("Deleting the local source file and local downloaded files");
Console.WriteLine();
File.Delete(sourceFile);
File.Delete(destinationFile);
}
}
else
{
Console.WriteLine(
"A connection string has not been defined in the system environment variables. " +
"Add a environment variable named 'storageconnectionstring' with your storage " +
"connection string as a value.");
}
}
/// <summary>
/// Task for deleting outdated analytics blobs that have reached older than 90 days
/// </summary>
/// <returns>Task<returns>
private static async Task SkycastAnalyticsDeletionTaskAsync()
{
DateTime startTime = DateTime.UtcNow;
// Retrieve storage account information from connection string
CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create a blob client for interacting with the blob service.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Create a container for organizing blobs within the storage account.
Console.WriteLine("1. Connect to Container\n ------------------------------");
//CloudBlobContainer container = blobClient.GetContainerReference("skycast-ife-analytics-device-outputs");
CloudBlobContainer container = blobClient.GetContainerReference("skycast-ife-analytics-device-outputs");
// List all blobs in this container. Because a container can contain a large number of blobs the results
// are returned in segments (pages) with a maximum of 5000 blobs per segment.
Console.WriteLine("2. Delete old Blobs in Container\n ------------------------------");
BlobContinuationToken token = null;
int totalCount = 0;
int deleteCount = 0;
do
{
BlobResultSegment resultSegment = await container.ListBlobsSegmentedAsync(token);
token = resultSegment.ContinuationToken;
DateTime currentDate = DateTime.UtcNow;
DateTime maxLastModified = currentDate.AddDays(-90);
long maxLastModifiedTicks = maxLastModified.Ticks;
//Uncomment for copying blobs to another container for testing
/**
* foreach (CloudBlockBlob blob in resultSegment.Results)
* {
* String downloadFileName = string.Format("./{0}", blob.Name);
* await blob.DownloadToFileAsync(downloadFileName, FileMode.Create);
* CloudBlockBlob blockBlob = testContainer.GetBlockBlobReference(blob.Name);
* await blockBlob.UploadFromFileAsync(downloadFileName);
*
* }**/
IEnumerable <IListBlobItem> toBeDeleted = resultSegment.Results.Where((b) => ((CloudBlockBlob)b).Properties.LastModified.Value.UtcTicks < maxLastModifiedTicks);
Console.WriteLine("Total - {0} - vs To Be Deleted - {1}", resultSegment.Results.Count(), toBeDeleted.Count());
foreach (CloudBlockBlob blob in toBeDeleted)
{
await blob.DeleteAsync();
Console.WriteLine(" - DELETED - {0} - LastModified: {1}", blob.Name, blob.Properties.LastModified.ToString());
}
totalCount = totalCount + resultSegment.Results.Count();
deleteCount = deleteCount + toBeDeleted.Count();
} while (token != null);
// Clean up after the demo
DateTime endTime = DateTime.UtcNow;
Console.WriteLine("\n\n------------------------------ SUMMARY ------------------------------");
Console.WriteLine("DELETION COMPLETE - Total Deleted: {0}", deleteCount);
Console.WriteLine("DELETION START TIME: {0}", startTime.ToLongTimeString());
Console.WriteLine("DELETION END TIME: {0}", endTime.ToLongTimeString());
Console.WriteLine("Total Number of Blobs before Deletion: {0}", totalCount);
Console.WriteLine("Total Number of Blobs after Deletion: {0}", totalCount - deleteCount);
Console.WriteLine("Total Time Elapsed - {0}", TimeSpan.FromTicks(endTime.Ticks - startTime.Ticks));
}
/// List part of blobs.
/// </summary>
/// <param name="prefix">Blob prefix</param>
/// <param name="useFlatBlobListing">Use flat blob listing</param>
/// <param name="blobListingDetails">Blob listing details.</param>
/// <param name="maxResults">Max results.</param>
/// <param name="currentToken">Current token.</param>
/// <param name="options">Request options</param>
/// <param name="operationContext">Operation Context.</param>
/// <returns>BlobResultSegment object</returns>
public BlobResultSegment ListBlobsSegmented(CloudBlobContainer container, string prefix, bool useFlatBlobListing,
BlobListingDetails blobListingDetails, int?maxResults, BlobContinuationToken currentToken, BlobRequestOptions options, OperationContext operationContext)
{
return(container.ListBlobsSegmented(prefix, useFlatBlobListing, blobListingDetails, maxResults, currentToken, options, operationContext));
}
/// <summary>
/// list blobs by blob prefix and container name
/// </summary>
/// <param name="containerName">container name</param>
/// <param name="prefix">blob preifx</param>
/// <returns>An enumerable collection of IListBlobItem</returns>
internal async Task ListBlobsByPrefix(long taskId, IStorageBlobManagement localChannel, string containerName, string prefix, Func <string, bool> blobFilter = null, bool includeDeleted = false, bool includeVersion = false)
{
CloudBlobContainer container = await GetCloudBlobContainerByName(localChannel, containerName).ConfigureAwait(false);
BlobContainerClient track2container = AzureStorageContainer.GetTrack2BlobContainerClient(container, localChannel.StorageContext, ClientOptions);
int listCount = InternalMaxCount;
int MaxListCount = 5000;
int requestCount = MaxListCount;
int realListCount = 0;
BlobContinuationToken continuationToken = ContinuationToken;
string track2ContinuationToken = this.ContinuationToken is null ? null : this.ContinuationToken.NextMarker;
if (UseTrack2Sdk()) // For new feature only available on Track2 SDK, need list with Track2 SDK.
{
BlobTraits blobTraits = BlobTraits.Metadata | BlobTraits.CopyStatus; // | BlobTraits.Tags;
BlobStates blobStates = BlobStates.Snapshots;
if (includeDeleted)
{
blobStates = blobStates | BlobStates.Deleted;
}
if (includeVersion)
{
blobStates = blobStates | BlobStates.Version;
}
if (IncludeTag.IsPresent)
{
blobTraits = blobTraits | BlobTraits.Tags;
}
do
{
requestCount = Math.Min(listCount, MaxListCount);
realListCount = 0;
IEnumerator <Page <BlobItem> > enumerator = track2container.GetBlobs(blobTraits, blobStates, prefix, CmdletCancellationToken)
.AsPages(track2ContinuationToken, requestCount)
.GetEnumerator();
Page <BlobItem> page;
enumerator.MoveNext();
page = enumerator.Current;
foreach (BlobItem item in page.Values)
{
if (blobFilter == null || blobFilter(item.Name))
{
OutputStream.WriteObject(taskId, GetAzureStorageBlob(item, track2container, localChannel.StorageContext, page.ContinuationToken, ClientOptions));
}
realListCount++;
}
track2ContinuationToken = page.ContinuationToken;
if (InternalMaxCount != int.MaxValue)
{
listCount -= realListCount;
}
} while (listCount > 0 && !string.IsNullOrEmpty(track2ContinuationToken));
}
else
{
BlobRequestOptions requestOptions = RequestOptions;
bool useFlatBlobListing = true;
BlobListingDetails details = BlobListingDetails.Snapshots | BlobListingDetails.Metadata | BlobListingDetails.Copy;
if (includeDeleted)
{
details = details | BlobListingDetails.Deleted;
}
do
{
requestCount = Math.Min(listCount, MaxListCount);
realListCount = 0;
BlobResultSegment blobResult = await localChannel.ListBlobsSegmentedAsync(container, prefix, useFlatBlobListing,
details, requestCount, continuationToken, requestOptions, OperationContext, CmdletCancellationToken).ConfigureAwait(false);
foreach (IListBlobItem blobItem in blobResult.Results)
{
CloudBlob blob = blobItem as CloudBlob;
if (blob == null)
{
continue;
}
if (blobFilter == null || blobFilter(blob.Name))
{
WriteCloudBlobObject(taskId, localChannel, blob, blobResult.ContinuationToken);
realListCount++;
}
}
if (InternalMaxCount != int.MaxValue)
{
listCount -= realListCount;
}
continuationToken = blobResult.ContinuationToken;
}while (listCount > 0 && continuationToken != null);
}
}
private void EnumerateContainers(string containerPrefix = "", bool testConnectivity = false)
{
BlobContinuationToken blobToken = new BlobContinuationToken();
ContainerResultSegment containerSegment = null;
string containerFilter = Config.ContainerFilter ?? string.Empty;
try
{
Log.Info("account sas");
while (blobToken != null)
{
Log.Info($"containerPrefix:{containerPrefix} containerFilter:{containerFilter}");
containerSegment = _blobClient.ListContainersSegmentedAsync(containerPrefix, blobToken).Result;
if (testConnectivity)
{
return;
}
if (containerSegment.Results.Any())
{
IEnumerable <CloudBlobContainer> containers = containerSegment.Results.Where(x => Regex.IsMatch(x.Name, containerFilter, RegexOptions.IgnoreCase));
Log.Info("container list results:", containers.Select(x => x.Name));
if (containers.Count() < 1)
{
Log.Warning($"no containers detected. use 'containerFilter' argument to specify");
}
if (containers.Count() > 1)
{
Log.Warning($"multiple containers detected. use 'containerFilter' argument to enumerate only one");
}
foreach (CloudBlobContainer container in containers)
{
AddContainerToList(container);
}
blobToken = containerSegment.ContinuationToken;
}
else if (!string.IsNullOrEmpty(containerPrefix))
{
Log.Warning("retrying without containerPrefix");
containerPrefix = null;
}
else
{
Log.Warning("no results");
break;
}
}
return;
}
catch (Exception e)
{
Log.Debug($"{e}");
Log.Warning($"unable to connect to containerPrefix: {containerPrefix} containerFilter: {containerFilter} error: {e.HResult}");
}
if (Config.SasEndpointInfo.AbsolutePath.Length > 1)
{
Log.Info("absolute path sas");
CloudBlobContainer container = new CloudBlobContainer(new Uri(_account.BlobEndpoint + Config.SasEndpointInfo.AbsolutePath + "?" + _account.Credentials.SASToken));
// force connection / error
if (container.ListBlobsSegmented(null, true, new BlobListingDetails(), 1, null, null, null).Results.Count() == 1)
{
if (testConnectivity)
{
return;
}
Log.Info($"connected with absolute path to container:{container.Name}", ConsoleColor.Green);
AddContainerToList(container);
}
else
{
string errMessage = $"there are no blobs in container:{container.Name}";
Log.Error(errMessage);
throw new Exception(errMessage);
}
}
else
{
string errMessage = "unable to enumerate containers with or without absolute path";
Log.Error(errMessage);
throw new Exception(errMessage);
}
}
public async Task <BlobElement[]> ListAsync(string path, CancellationToken cancellationToken = default)
{
var pathSegments = PathUtilities.GetSegments(path);
if (pathSegments.Length == 0)
{
var cloudBlobClient = _storageAccount.CreateCloudBlobClient();
BlobContinuationToken continuationToken = null;
var results = new List <BlobElement>();
do
{
var response = await cloudBlobClient.ListContainersSegmentedAsync(continuationToken).ConfigureAwait(false);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results.Select(c => new BlobElement(c.Name, c.Name, BlobElementType.Container)));
}while (continuationToken != null);
return(results.ToArray());
}
else
{
var containerName = pathSegments.First();
var containerPath = string.Join(PathUtilities.Delimiter, pathSegments.Skip(1));
var container = await GetCloudBlobContainerAsync(containerName, cancellationToken).ConfigureAwait(false);
BlobContinuationToken continuationToken = null;
var results = new List <BlobElement>();
do
{
var response = pathSegments.Skip(1).Any() ?
await container.ListBlobsSegmentedAsync(containerPath + PathUtilities.Delimiter, continuationToken).ConfigureAwait(false) :
await container.ListBlobsSegmentedAsync(continuationToken).ConfigureAwait(false);
continuationToken = response.ContinuationToken;
#pragma warning disable CS8620 // Argument cannot be used for parameter due to differences in the nullability of reference types.
results.AddRange(response.Results.Select(i =>
{
if (i is CloudBlobDirectory directory)
{
return(new BlobElement(directory.Prefix, PathUtilities.GetSegments(directory.Prefix).Last(), BlobElementType.Container));
}
else if (i is CloudBlob blob)
{
var blobNameSegments = blob.Name.Split(PathUtilities.DelimiterChar);
return(new BlobElement(
blob.Name,
string.Join(PathUtilities.Delimiter, blobNameSegments.Skip(blobNameSegments.Length - 1)),
BlobElementType.Blob,
blob.Properties.Length,
blob.Properties.Created,
blob.Properties.LastModified));
}
else
{
return(null);
}
}).Where(c => c != null));
#pragma warning restore CS8620 // Argument cannot be used for parameter due to differences in the nullability of reference types.
}while (continuationToken != null);
if (results.Count == 0)
{
throw new ContainerNotFoundException(path, null);
}
return(results.ToArray());
}
}
/// <summary>
/// Sync to local
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public override async Task <IEnumerable <FolderItemSyncResult> > SyncFolderAsync(CancellationToken cancellationToken)
{
BlobContinuationToken blobContinuationToken = null;
var ret = new List <FolderItemSyncResult>();
using (var semaphoreSlim = new SemaphoreSlim(Parallel))
{
var tasks = new List <Task>();
while (!cancellationToken.IsCancellationRequested)
{
var blobs = await Container.ListBlobsSegmentedAsync(Prefix,
true,
BlobListingDetails.None,
null,
blobContinuationToken, null, null, cancellationToken).ConfigureAwait(false);
blobContinuationToken = blobs.ContinuationToken;
foreach (var blob in blobs.Results)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (!(blob is CloudBlockBlob cloudBlockBlob))
{
continue;
}
var nameWithoutPrefix = cloudBlockBlob.Name.Substring(Prefix.Length);
var localPath = Path.Combine(TargetLocalFolder, nameWithoutPrefix);
var fileInfo = new FileInfo(localPath);
var diff = false;
if (!fileInfo.Exists)
{
var directory = fileInfo.Directory;
diff = true;
if (directory != null && !directory.Exists)
{
directory.Create();
}
}
else
{
if (fileInfo.LastWriteTimeUtc < cloudBlockBlob.Properties.LastModified)
{
diff = true;
}
}
if (diff)
{
await semaphoreSlim.WaitAsync(cancellationToken).ConfigureAwait(false);
var downloadTask = cloudBlockBlob.DownloadToFileAsync(localPath, FileMode.Create, null, null, null, null, cancellationToken)
.ContinueWith(result =>
{
if (result.Status == TaskStatus.RanToCompletion)
{
if (!fileInfo.Exists)
{
fileInfo = new FileInfo(localPath);
}
ret.Add(new FolderItemSyncResult()
{
Path = nameWithoutPrefix,
LastModified = fileInfo.LastWriteTimeUtc,
Ex = null,
Result = FolderItemSyncResultEnum.UpdateSuccess
});
}
else
{
try
{
if (File.Exists(localPath))
{
File.Delete(localPath);
}
}
catch
{
//
}
ret.Add(new FolderItemSyncResult()
{
Path = nameWithoutPrefix,
LastModified = fileInfo.LastWriteTimeUtc,
Ex = result.Exception,
Result = FolderItemSyncResultEnum.UpdateFailure
});
}
semaphoreSlim.Release();
});
tasks.Add(downloadTask);
}
else
{
ret.Add(new FolderItemSyncResult()
{
Path = nameWithoutPrefix,
LastModified = fileInfo.LastWriteTimeUtc,
Ex = null,
Result = FolderItemSyncResultEnum.Skip
});
}
}
if (blobContinuationToken == null)
{
break;
}
}
await Task.WhenAll(tasks).ConfigureAwait(false);
return(ret);
}
}
/// <summary>
/// Async get container permission
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="taskId">Task id</param>
/// <param name="context">Azure storage context</param>
/// <returns></returns>
internal async Task GetContainerPermission(long taskId, IStorageBlobManagement localChannel, CloudBlobContainer container, BlobContinuationToken continuationToken)
{
BlobRequestOptions requestOptions = RequestOptions;
AccessCondition accessCondition = null;
BlobContainerPermissions permissions = null;
try
{
permissions = await localChannel.GetContainerPermissionsAsync(container, accessCondition,
requestOptions, OperationContext, CmdletCancellationToken);
}
catch (StorageException e)
{
if (!e.IsNotFoundException())
{
throw;
}
//404 Not found means we don't have permission to query the Permission of the specified container.
}
WriteCloudContainerObject(taskId, localChannel, container, permissions, continuationToken);
}
