private async static void UpdateCertainBlock()
{
try
{
CloudBlobContainer container = _cloudBlobClient.GetContainerReference("text-files");
if (await container.ExistsAsync())
{
CloudBlockBlob fileBlob = container.GetBlockBlobReference("sample.txt");
if (await fileBlob.ExistsAsync())
{
List <ListBlockItem> ids = new List <ListBlockItem>(await fileBlob.DownloadBlockListAsync());
if (ids.Count > 0)
{
string blockId = "0x002";
string blockIdBase64 = Convert.ToBase64String(UTF8Encoding.UTF8.GetBytes(blockId));
string newContent = "000000000" + Environment.NewLine;
List <string> idList = new List <string>();
if (ids.Any(x => x.Name.Equals(blockIdBase64, StringComparison.InvariantCultureIgnoreCase)))
{
await fileBlob.PutBlockAsync(blockIdBase64, new MemoryStream(UTF8Encoding.UTF8.GetBytes(newContent), true), null);
idList.AddRange(ids.Select(x => x.Name));
}
else
{
await fileBlob.PutBlockAsync(blockIdBase64, new MemoryStream(UTF8Encoding.UTF8.GetBytes(newContent), true), null);
idList.AddRange(ids.Select(x => x.Name));
idList.Add(blockIdBase64);
Console.WriteLine("Target block is not found. Additional Block has been added");
}
await fileBlob.PutBlockListAsync(idList);
Console.WriteLine("Done");
}
else
{
Console.WriteLine("No available blocks in the file");
}
}
else
{
Console.WriteLine("File doesn't exist");
}
}
else
{
Console.WriteLine("Container doesn't exist");
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
private static async Task AppendCommitInternalAsync(CloudBlockBlob blob, string[] blockIds, CancellationToken cancellationToken)
{
IEnumerable <ListBlockItem> blockList = await blob.DownloadBlockListAsync(
BlockListingFilter.Committed,
accessCondition : null,
options : null,
operationContext : null,
cancellationToken);
List <string> newBlockLists = blockList.Select(b => b.Name).ToList();
newBlockLists.AddRange(blockIds);
await CommitInternalAsync(blob, newBlockLists.ToArray(), cancellationToken);
}
/// <summary>
/// Reads the blob's block list, and indicates whether the blob has been committed.
/// </summary>
/// <param name="blob">A CloudBlockBlob object.</param>
/// <returns>A Task object.</returns>
private static async Task ReadBlockListAsync(CloudBlockBlob blob)
{
// Get the blob's block list.
foreach (var listBlockItem in await blob.DownloadBlockListAsync(BlockListingFilter.All, null, null, null))
{
Console.WriteLine(
listBlockItem.Committed
? "Block {0} has been committed to block list. Block length = {1}"
: "Block {0} is uncommitted. Block length = {1}",
listBlockItem.Name,
listBlockItem.Length);
}
Console.WriteLine();
}
/// <summary>
/// Waits for all expected blocks to finish uploading, commits the blob, then deletes the blob and container used during the test.
/// </summary>
/// <remarks>This method will only be called if application is running as ID 0.</remarks>
/// <param name="container">The blob's container.</param>
/// <param name="blobName">The name of the blob being uploaded.</param>
/// <param name="totalBlocks">The total number of blocks to commit to the blob.</param>
private static async Task FinalizeBlob(CloudBlobContainer container, string blobName, uint totalBlocks, bool cleanup)
{
// Define the order in which to commit blocks.
List <string> blockIdList = new List <string>();
for (uint i = 0; i < totalBlocks; ++i)
{
blockIdList.Add(Convert.ToBase64String(BitConverter.GetBytes(i)));
}
HashSet <string> blockIdSet = new HashSet <string>(blockIdList);
CloudBlockBlob blockblob = container.GetBlockBlobReference(blobName);
// Poll the blob's Uncommitted Block List until we determine that all expected blocks have been successfully uploaded.
Console.WriteLine("Waiting for all expected blocks to appear in the uncommitted block list.");
IEnumerable <ListBlockItem> currentUncommittedBlockList = new List <ListBlockItem>();
while (true)
{
currentUncommittedBlockList = await blockblob.DownloadBlockListAsync(BlockListingFilter.Uncommitted, AccessCondition.GenerateEmptyCondition(), null, null);
if (currentUncommittedBlockList.Count() >= blockIdList.Count &&
VerifyBlocks(currentUncommittedBlockList, blockIdSet))
{
break;
}
Console.WriteLine($"{currentUncommittedBlockList.Count()} / {blockIdList.Count} blocks in the uncommitted block list.");
await Task.Delay(TimeSpan.FromSeconds(5));
}
Console.WriteLine("Issuing PutBlockList.");
await blockblob.PutBlockListAsync(blockIdList);
if (cleanup)
{
Console.WriteLine("Press 'Enter' key to delete the example container and blob.");
Console.ReadLine();
Console.WriteLine("Issuing DeleteBlob.");
await blockblob.DeleteAsync();
Console.WriteLine("Deleting container.");
await container.DeleteAsync();
}
}
public async Task <int> GetBlockListAsync()
{
StorageCredentials creds = new StorageCredentials(AcctName, AcctKey);
CloudStorageAccount account = new CloudStorageAccount(creds, useHttps: true);
CloudBlobClient client = account.CreateCloudBlobClient();
CloudBlockBlob blob = new CloudBlockBlob(Uri, client);
System.Collections.Generic.IEnumerable <ListBlockItem> list = await blob.DownloadBlockListAsync();
int count = 0;
foreach (var item in list)
{
count++;
}
return(count);
}
public void UploadBlock(Stream blockStream, string containerName, string fileName, int blockIndex, int blocksCount, string contentType)
{
string blockId = Convert.ToBase64String(BitConverter.GetBytes(blockIndex));
CloudBlockBlob blob = GetBlockBlob(containerName, null, fileName);
blob.PutBlockAsync(blockId, blockStream, null).Wait();
if (blockIndex == blocksCount - 1)
{
BlockListingFilter blockType = BlockListingFilter.Uncommitted;
IEnumerable <ListBlockItem> blockList = blob.DownloadBlockListAsync(blockType, null, null, null).Result;
List <string> blockIds = new List <string>();
foreach (ListBlockItem blockItem in blockList)
{
blockIds.Add(blockItem.Name);
}
blob.PutBlockListAsync(blockIds).Wait();
blob.Properties.ContentType = contentType;
blob.SetPropertiesAsync().Wait();
}
}
internal async static Task <IEnumerable <ListBlockItem> > GetBlockListForBlob(CloudBlockBlob destBlob, int retryCount, ILogger logger)
{
IEnumerable <ListBlockItem> retVal = null;
// use retry policy which will automatically handle the throttling related StorageExceptions
await GetStorageRetryPolicy($"DownloadBlockListAsync for blob {destBlob.Name}", retryCount, logger).ExecuteAsync(async() =>
{
if (!await destBlob.ExistsAsync())
{
// obvious but setting explicitly
retVal = null;
}
else
{
retVal = await destBlob.DownloadBlockListAsync(BlockListingFilter.Committed, null, null, null);
}
});
return(retVal);
}
/// <summary>
/// Writes data from a stream to a blob.
/// </summary>
/// <param name="fileStream">The stream to read from.</param>
/// <param name="connectionString">The connection string for the destination blob storage.</param>
/// <param name="containerName">The name of the container to place the blob.</param>
/// <param name="destinationPath">The relative path (from the container) for the blob.</param>
/// <param name="createContainerIfNotExists">Specifies if the container should be created if it does not exist.</param>
/// <returns>A <see cref="Microsoft.AdvocacyPlatform.Contracts.FileInfo"/> object representing the new blob.</returns>
public async Task <Contracts.FileInfo> WriteStreamAsync(Stream fileStream, Secret connectionString, string containerName, string destinationPath, bool createContainerIfNotExists)
{
try
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString.Value);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
if (createContainerIfNotExists)
{
await container.CreateIfNotExistsAsync();
}
CloudBlockBlob blockBlob = container.GetBlockBlobReference(destinationPath);
bool exists = await blockBlob.ExistsAsync();
if (exists)
{
throw new RecordingExistsException(destinationPath);
}
await blockBlob.UploadFromStreamAsync(fileStream);
IEnumerable <ListBlockItem> blockBlobBlocks = await blockBlob.DownloadBlockListAsync();
return(new Contracts.FileInfo()
{
SizeInBytes = blockBlobBlocks.Sum(x => x.Length),
Uri = blockBlob.Uri,
});
}
catch (StorageException ex)
{
throw new StorageClientException("Storage client encountered exception", ex);
}
}
//
// Function for calculating the size of a blob
//
private static void CalculateBlobSize(string accountName, object containerName, object blobName, string sasToken, bool isDisk)
{
// Now that we hvae the SAS-token, let's calculate the size of the snapshot
var blobSizeInBytes = 0F;
System.Console.WriteLine("Trying to retrieve Blob...");
var blob = default(ICloudBlob);
var blobUri = new Uri($"https://{accountName}.blob.core.windows.net/{containerName}/{blobName}?{sasToken}");
try
{
var reqOptions = new BlobRequestOptions
{
ServerTimeout = TimeSpan.FromMinutes(5),
MaximumExecutionTime = TimeSpan.FromMinutes(5)
};
if (isDisk)
{
var pageBlob = new CloudPageBlob(blobUri);
blob = pageBlob;
Task.WaitAll(blob.FetchAttributesAsync());
var blobSize = pageBlob.Properties.Length;
System.Console.WriteLine($"Blob size per API in bytes: {blobSize}.");
System.Console.WriteLine($"Blob size per API in MB: {blobSize / 1024 / 1024}.");
System.Console.WriteLine($"Blob size per API in GB: {blobSize / 1024 / 1024 / 1024}.");
var pageRanges = pageBlob.GetPageRangesAsync().Result;
foreach (var rg in pageRanges)
{
blobSizeInBytes += (12 + (rg.EndOffset - rg.StartOffset));
}
}
else
{
var blockBlob = new CloudBlockBlob(blobUri);
blob = blockBlob;
blobSizeInBytes += 8;
var blockList = blockBlob.DownloadBlockListAsync().Result;
foreach (var bl in blockList)
{
blobSizeInBytes += bl.Length + bl.Name.Length;
}
}
}
catch (Exception ex)
{
System.Console.WriteLine($"EXCPETION: {ex.Message}");
if (ex.InnerException != null)
{
System.Console.WriteLine($"INNER EXCEPTION: {ex.InnerException.Message}");
}
}
// Add the base size to the previously calculated size
blobSizeInBytes += (124 + (blob.Name.Length * 2));
foreach (var md in blob.Metadata)
{
blobSizeInBytes += (3 + (md.Key.Length + md.Value.Length));
}
// Output the final size of the blob
System.Console.WriteLine("");
System.Console.WriteLine($"Blob Size per calculation in bytes: {blobSizeInBytes}");
System.Console.WriteLine($"Blob Size per calculation in MB: {blobSizeInBytes / 1024 / 1024}");
System.Console.WriteLine($"Blob Size per calculation in GB: {blobSizeInBytes / 1024 / 1024 / 1024}");
}
static void Main(string[] args)
{
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json");
Configuration = builder.Build();
CloudStorageAccount account = CloudStorageAccount.Parse(Configuration["AppSettings:storageConnection"]);
CloudQueueClient queueClient = account.CreateCloudQueueClient();
CloudQueue queue = queueClient.GetQueueReference("blobcreateevents");
queue.CreateIfNotExistsAsync();
CloudBlobClient blobClient = account.CreateCloudBlobClient();
bool done = false;
ListBlockItem lastBlockDownloaded = null;
int currentBackOff = 0;
int maxBackOff = 5;
byte[] bytesFromBlob = new byte[4 * 1024 * 1024];
do
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Getting messages from the queue.");
queue.FetchAttributesAsync().Wait();
int countOfMessages = Convert.ToInt32(queue.ApproximateMessageCount);
if (countOfMessages > 0)
{
currentBackOff = 1;
var msgs = queue.GetMessagesAsync(countOfMessages > 32 ? 32 : countOfMessages).Result;
var numberOfMessagesLeftInBatch = msgs.Count();
foreach (var msg in msgs)
{
if (numberOfMessagesLeftInBatch == 1)
// process only the last message
// all messages are equivalent in meaning
// sender usually gets ahead of receiver
{
JToken token = JObject.Parse(msg.AsString);
string resourceId = (string)token.SelectToken("topic");
string subject = (string)token.SelectToken("subject");
string eventType = (string)token.SelectToken("eventType");
if (eventType == "Microsoft.Storage.BlobCreated")
{
var storageAccountName = resourceId.Split('/')[8];
var storageContainerName = subject.Split('/')[4];
var blobName = subject.Split('/')[6];
Uri uri = new Uri($"https://{storageAccountName}.blob.core.windows.net/{storageContainerName}/{blobName}");
CloudBlockBlob blob = new CloudBlockBlob(uri, blobClient);
var blockList = blob.DownloadBlockListAsync().Result;
long blobOffset = 0;
var blocksToCopy = blockList;
if (lastBlockDownloaded != null)
{
// count the blocks already written
var countOfblocksAlreadyWritten = blockList.TakeWhile(item => (item.Name != lastBlockDownloaded.Name)).Count() + 1;
// get an enumerable of those block list items
var blocksAlreadyWritten = blockList.Take(countOfblocksAlreadyWritten);
// add up the bytes already written
foreach (var block in blocksAlreadyWritten)
{
blobOffset += block.Length;
}
// skip over blocks already written
blocksToCopy = blockList.SkipWhile(item => (item.Name != lastBlockDownloaded.Name)).Skip(1);
}
if (blocksToCopy.Count() > 0)
{
var fs = File.OpenWrite(@"c:\temp\abigfile.dat");
using (BinaryWriter writer = new BinaryWriter(fs))
{
foreach (var block in blocksToCopy)
{
blob.DownloadRangeToByteArrayAsync(bytesFromBlob, 0, blobOffset, block.Length).Wait();
writer.Seek(0, SeekOrigin.End);
writer.Write(bytesFromBlob, 0, (int)block.Length);
blobOffset += block.Length;
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine($"{block.Name}");
}
};
fs.Close();
}
lastBlockDownloaded = blockList.Last();
}
}
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Deleting a message from the queue");
queue.DeleteMessageAsync(msg);
numberOfMessagesLeftInBatch--;
}
}
else
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"Waiting for {currentBackOff} seconds for next message");
Thread.Sleep(currentBackOff * 1000);
currentBackOff += (currentBackOff < maxBackOff ? 1 : 0);
}
// set done flag in some way, or loop forever.
} while (!done);
Console.ReadKey();
}
public static async Task <string> UploadAsync(Func <long, int, Task <byte[]> > fetchLocalData, long blobLenth,
CloudBlockBlob blockBlob, uint uploadParallelism = DEFAULT_PARALLELISM)
{
const int MAXIMUM_UPLOAD_SIZE = 4 * MB;
if (NumBytesPerChunk > MAXIMUM_UPLOAD_SIZE)
{
NumBytesPerChunk = MAXIMUM_UPLOAD_SIZE;
}
#region Which blocks exist in the file
var allBlockInFile = Enumerable
.Range(0, 1 + ((int)(blobLenth / NumBytesPerChunk)))
.Select(_ => new BlockMetadata(_, blobLenth, NumBytesPerChunk))
.Where(block => block.Length > 0)
.ToList();
var blockIdList = allBlockInFile.Select(_ => _.BlockId).ToList();
#endregion
#region Which blocks are already uploaded
List <BlockMetadata> missingBlocks = null;
try
{
var existingBlocks = (await blockBlob.DownloadBlockListAsync(
BlockListingFilter.Uncommitted,
AccessCondition.GenerateEmptyCondition(),
new BlobRequestOptions {
},
new OperationContext {
}))
.Where(_ => _.Length == NumBytesPerChunk)
.ToList();
missingBlocks = allBlockInFile.Where(blockInFile => !existingBlocks.Any(existingBlock =>
existingBlock.Name == blockInFile.BlockId &&
existingBlock.Length == blockInFile.Length)).ToList();
}
catch (StorageException)
{
missingBlocks = allBlockInFile;
}
#endregion
Func <BlockMetadata, Statistics, Task> uploadBlockAsync = async(block, stats) =>
{
byte[] blockData = await fetchLocalData(block.Index, block.Length);
string contentHash = md5()(blockData);
DateTime start = DateTime.UtcNow;
await ExecuteUntilSuccessAsync(async() =>
{
await blockBlob.PutBlockAsync(
blockId: block.BlockId,
blockData: new MemoryStream(blockData, true),
contentMD5: contentHash,
accessCondition: AccessCondition.GenerateEmptyCondition(),
options: new BlobRequestOptions
{
StoreBlobContentMD5 = true,
UseTransactionalMD5 = true
},
operationContext: new OperationContext());
}, consoleExceptionHandler);
stats.Add(block.Length, start);
};
var s = new Statistics(missingBlocks.Sum(b => (long)b.Length));
await LargeFileUploaderUtils.ForEachAsync(
source : missingBlocks,
parallelUploads : 4,
body : blockMetadata => uploadBlockAsync(blockMetadata, s));
await ExecuteUntilSuccessAsync(async() =>
{
await blockBlob.PutBlockListAsync(blockIdList);
}, consoleExceptionHandler);
log("PutBlockList succeeded, finished upload to {0}", blockBlob.Uri.AbsoluteUri);
return(blockBlob.Uri.AbsoluteUri);
}
public static IEnumerable <ListBlockItem> DownloadBlockList(this CloudBlockBlob blob, BlockListingFilter blockListingFilter = BlockListingFilter.Committed, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null)
{
return(blob.DownloadBlockListAsync(blockListingFilter, accessCondition, options, operationContext).GetAwaiter().GetResult());
}
/// <summary>Appends a block to the end of a block blob.</summary>
/// <param name="blob">The blob to append a block to.</param>
/// <param name="info">Additional info used when the blob is first being created.</param>
/// <param name="dataToAppend">The data to append as a block to the end of the blob.</param>
/// <param name="options">The blob request options.</param>
/// <param name="operationContext">The operation context.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A Task indicating when the operation has completed.</returns>
public static async Task AppendBlockAsync(this CloudBlockBlob blob, AppendBlobBlockInfo info,
Byte[] dataToAppend, BlobRequestOptions options = null, OperationContext operationContext = null,
CancellationToken cancellationToken = default(CancellationToken))
{
blob.Properties.ContentType = info.MimeType;
//blob.Properties.ContentDisposition = "attachment; filename=\"fname.ext\"";
// NOTE: Put Block ignores access condition and so it always succeeds
List <String> blockList = new List <String>();
while (true)
{
blockList.Clear();
AccessCondition accessCondition;
try {
blockList.AddRange((await blob.DownloadBlockListAsync(BlockListingFilter.Committed,
null, options, operationContext, cancellationToken).ConfigureAwait(false)).Select(lbi => lbi.Name)); // 404 if blob not found
accessCondition = AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag); // Write if blob matches what we read
}
catch (StorageException se) {
if (!se.Matches(HttpStatusCode.NotFound, BlobErrorCodeStrings.BlobNotFound))
{
throw;
}
accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); // Write if blob doesn't exist
}
try {
if (blockList.Count == 0) // Blob doesn't exist yet, add header (if specified)
// Notify client code that new blob is about to be created
{
info.OnCreatingBlob(blob);
if (info.Header != null) // Write header to new blob
{
String headerBlockId = Guid.NewGuid().ToString().Encode().ToBase64String();
await blob.PutBlockAsync(headerBlockId, new MemoryStream(info.Header), null).ConfigureAwait(false);
blockList.Add(headerBlockId);
}
}
// Upload new block & add it's Id to the block list
String blockId = Guid.NewGuid().ToString().Encode().ToBase64String();
await blob.PutBlockAsync(blockId, new MemoryStream(dataToAppend), null).ConfigureAwait(false);
blockList.Add(blockId);
// If too many blocks, remove old block (but not the header if it exists)
var maxEntries = info.MaxEntries + ((info.Header == null) ? 0 : 1);
if (blockList.Count > maxEntries)
{
blockList.RemoveAt((info.Header == null) ? 0 : 1);
}
// Upload the new block list
await blob.PutBlockListAsync(blockList, AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag),
options, operationContext, cancellationToken).ConfigureAwait(false); // 409 if blob created behind our back; 400 if block Id doesn't exist (happens in another PC calls PutBlockList after our PutBlock)
break; // If successful, we're done; don't retry
}
catch (StorageException se) {
// Blob got created behind our back, retry
if (se.Matches(HttpStatusCode.Conflict, BlobErrorCodeStrings.BlobAlreadyExists))
{
continue;
}
// Blob got created or modified behind our back, retry
if (se.Matches(HttpStatusCode.PreconditionFailed))
{
continue;
}
// Another PC called PutBlockList between our PutBlock & PutBlockList,
// our block(s) got destroyed, retry
if (se.Matches(HttpStatusCode.BadRequest, BlobErrorCodeStrings.InvalidBlockList))
{
continue;
}
throw;
}
}
}
public static async Task <string> UploadAsync(Func <long, int, Task <byte[]> > fetchLocalData, long blobLenth,
CloudBlockBlob blockBlob, uint uploadParallelism = DEFAULT_PARALLELISM)
{
const int MAXIMUM_UPLOAD_SIZE = 100 * MB;
if (NumBytesPerChunk > MAXIMUM_UPLOAD_SIZE)
{
NumBytesPerChunk = MAXIMUM_UPLOAD_SIZE;
}
#region Which blocks exist in the file
var allBlockInFile = Enumerable
.Range(0, 1 + ((int)(blobLenth / NumBytesPerChunk)))
.Select(_ => new BlockMetadata(_, blobLenth, NumBytesPerChunk))
.Where(block => block.Length > 0)
.ToList();
var blockIdList = allBlockInFile.Select(_ => _.BlockId).ToList();
#endregion
#region Which blocks are already uploaded
List <BlockMetadata> missingBlocks = null;
try
{
var existingBlocks = (await blockBlob.DownloadBlockListAsync(
BlockListingFilter.Uncommitted,
AccessCondition.GenerateEmptyCondition(),
new BlobRequestOptions {
},
new OperationContext {
}))
.Where(_ => _.Length == NumBytesPerChunk)
.ToList();
missingBlocks = allBlockInFile.Where(blockInFile => !existingBlocks.Any(existingBlock =>
existingBlock.Name == blockInFile.BlockId &&
existingBlock.Length == blockInFile.Length)).ToList();
}
catch (StorageException)
{
missingBlocks = allBlockInFile;
}
#endregion
Func <BlockMetadata, Statistics, Task> uploadBlockAsync = async(block, stats) =>
{
DateTime start = DateTime.UtcNow;
stats.Add(block.Length, start);
var fetchLocalDataResult = await Policy
.Handle <Exception>()
.WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)))
.ExecuteAndCaptureAsync <byte[]>(() => fetchLocalData(block.Index, block.Length));
if (fetchLocalDataResult.Outcome == OutcomeType.Failure)
{
throw new Exception($"Could not read local file", fetchLocalDataResult.FinalException);
}
byte[] blockData = fetchLocalDataResult.Result;
string contentHash = md5()(blockData);
var putBlockAsyncResult = await Policy
.Handle <Exception>()
.WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)))
.ExecuteAndCaptureAsync(() => blockBlob.PutBlockAsync(
blockId: block.BlockId,
blockData: new MemoryStream(blockData, true),
contentMD5: contentHash,
accessCondition: AccessCondition.GenerateEmptyCondition(),
options: new BlobRequestOptions {
StoreBlobContentMD5 = true, UseTransactionalMD5 = true
},
operationContext: new OperationContext()));
if (putBlockAsyncResult.Outcome == OutcomeType.Failure)
{
throw new Exception($"Could not call PutBlockAsync {putBlockAsyncResult.FinalException.Message}", putBlockAsyncResult.FinalException);
}
};
var s = new Statistics(missingBlocks.Sum(b => (long)b.Length));
await LargeFileUploaderUtils.ForEachAsync(
source : missingBlocks,
parallelTasks : 4,
task : blockMetadata => uploadBlockAsync(blockMetadata, s));
var putBlockListAsyncResult = await Policy
.Handle <Exception>()
.WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)))
.ExecuteAndCaptureAsync(() => blockBlob.PutBlockListAsync(blockIdList));
if (putBlockListAsyncResult.Outcome == OutcomeType.Failure)
{
throw new Exception($"Could not call PutBlockListAsync", putBlockListAsyncResult.FinalException);
}
log("PutBlockList succeeded, finished upload to {0}", blockBlob.Uri.AbsoluteUri);
return(blockBlob.Uri.AbsoluteUri);
}
public static async Task Run(
[BlobTrigger("%blobContainerName%/resourceId=/SUBSCRIPTIONS/{subId}/RESOURCEGROUPS/{resourceGroup}/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/{nsgName}/y={blobYear}/m={blobMonth}/d={blobDay}/h={blobHour}/m={blobMinute}/macAddress={mac}/PT1H.json", Connection = "nsgSourceDataConnection")] CloudBlockBlob myBlob,
[Table("checkpoints", Connection = "AzureWebJobsStorage")] CloudTable checkpointTable,
Binder nsgDataBlobBinder,
Binder cefLogBinder,
string subId, string resourceGroup, string nsgName, string blobYear, string blobMonth, string blobDay, string blobHour, string blobMinute, string mac,
ExecutionContext executionContext,
ILogger log)
{
log.LogDebug($"BlobTriggerIngestAndTransmit triggered: {executionContext.InvocationId} ");
string nsgSourceDataAccount = Util.GetEnvironmentVariable("nsgSourceDataAccount");
if (nsgSourceDataAccount.Length == 0)
{
log.LogError("Value for nsgSourceDataAccount is required.");
throw new System.ArgumentNullException("nsgSourceDataAccount", "Please provide setting.");
}
string blobContainerName = Util.GetEnvironmentVariable("blobContainerName");
if (blobContainerName.Length == 0)
{
log.LogError("Value for blobContainerName is required.");
throw new System.ArgumentNullException("blobContainerName", "Please provide setting.");
}
string outputBinding = Util.GetEnvironmentVariable("outputBinding");
if (outputBinding.Length == 0)
{
log.LogError("Value for outputBinding is required. Permitted values are: 'arcsight', 'splunk', 'eventhub'.");
throw new System.ArgumentNullException("outputBinding", "Please provide setting.");
}
var blobDetails = new BlobDetails(subId, resourceGroup, nsgName, blobYear, blobMonth, blobDay, blobHour, blobMinute, mac);
// get checkpoint
Checkpoint checkpoint = Checkpoint.GetCheckpoint(blobDetails, checkpointTable);
var blockList = myBlob.DownloadBlockListAsync().Result;
var startingByte = blockList.Where((item, index) => index < checkpoint.CheckpointIndex).Sum(item => item.Length);
var endingByte = blockList.Where((item, index) => index < blockList.Count() - 1).Sum(item => item.Length);
var dataLength = endingByte - startingByte;
log.LogDebug("Blob: {0}, starting byte: {1}, ending byte: {2}, number of bytes: {3}", blobDetails.ToString(), startingByte, endingByte, dataLength);
if (dataLength == 0)
{
log.LogWarning(string.Format("Blob: {0}, triggered on completed hour.", blobDetails.ToString()));
return;
}
//foreach (var item in blockList)
//{
// log.LogInformation("Name: {0}, Length: {1}", item.Name, item.Length);
//}
var attributes = new Attribute[]
{
new BlobAttribute(string.Format("{0}/{1}", blobContainerName, myBlob.Name)),
new StorageAccountAttribute(nsgSourceDataAccount)
};
string nsgMessagesString = "";
var bytePool = ArrayPool <byte> .Shared;
byte[] nsgMessages = bytePool.Rent((int)dataLength);
try
{
CloudBlockBlob blob = nsgDataBlobBinder.BindAsync <CloudBlockBlob>(attributes).Result;
await blob.DownloadRangeToByteArrayAsync(nsgMessages, 0, startingByte, dataLength);
if (nsgMessages[0] == ',')
{
nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages, 1, (int)(dataLength - 1));
}
else
{
nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages, 0, (int)dataLength);
}
}
catch (Exception ex)
{
log.LogError(string.Format("Error binding blob input: {0}", ex.Message));
throw ex;
}
finally
{
bytePool.Return(nsgMessages);
}
//log.LogDebug(nsgMessagesString);
try
{
int bytesSent = await Util.SendMessagesDownstreamAsync(nsgMessagesString, executionContext, cefLogBinder, log);
log.LogInformation($"Sending {nsgMessagesString.Length} bytes (denormalized to {bytesSent} bytes) downstream via output binding {outputBinding}.");
}
catch (Exception ex)
{
log.LogError(string.Format("SendMessagesDownstreamAsync: Error {0}", ex.Message));
throw ex;
}
checkpoint.PutCheckpoint(checkpointTable, blockList.Count() - 1);
}
private static void CreateStreamAndUploadToBlob(string sourceFile, CloudBlockBlob destinationBlob)
{
#if DotNetCoreClr
if (destinationBlob.ExistsAsync().Result&& destinationBlob.DownloadBlockListAsync().Result.Any())
{
// Delete all blocks in case we crashed and failed the upload in previous attempt.
destinationBlob.PutBlockListAsync(new List <string>()).Wait();
}
int blockSize = 4 * 1024 * 1024; //4 MB
using (FileStream fileStream =
new FileStream(sourceFile, FileMode.Open, FileAccess.Read))
{
var tasks = new List <Task>();
long fileSize = fileStream.Length;
//block count is the number of blocks + 1 for the last one
int blockCount = (int)((float)fileSize / (float)blockSize) + 1;
//List of block ids; the blocks will be committed in the order of this list
List <string> blockIDs = new List <string>();
//starting block number
int blockNumber = 0;
int bytesRead = 0; //number of bytes read so far
long bytesLeft = fileSize; //number of bytes left to read and upload
//do until all of the bytes are uploaded
while (bytesLeft > 0)
{
blockNumber++;
int bytesToRead;
if (bytesLeft >= blockSize)
{
//more than one block left, so put up another whole block
bytesToRead = blockSize;
}
else
{
//less than one block left, read the rest of it
bytesToRead = (int)bytesLeft;
}
//create a blockID from the block number, add it to the block ID list
//the block ID is a base64 string
string blockId =
Convert.ToBase64String(System.Text.ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}",
blockNumber.ToString("d7"))));
blockIDs.Add(blockId);
//set up new buffer with the right size, and read that many bytes into it
byte[] bytes = new byte[bytesToRead];
fileStream.Read(bytes, 0, bytesToRead);
//calculate the MD5 hash of the byte array
string blockHash = GetMD5HashFromStream(bytes);
//upload the block, provide the hash so Azure can verify it
tasks.Add(destinationBlob.PutBlockAsync(blockId, new MemoryStream(bytes), blockHash));
//increment/decrement counters
bytesRead += bytesToRead;
bytesLeft -= bytesToRead;
// Do not start more instances to limit memory usage
if (tasks.Count == 5)
{
Task.WaitAll(tasks.ToArray());
tasks.Clear();
}
}
if (tasks.Any())
{
Task.WaitAll(tasks.ToArray());
}
//commit the blocks
destinationBlob.PutBlockListAsync(blockIDs).Wait();
}
#else
using (FileStream fs = new FileStream(sourceFile, FileMode.Open, FileAccess.Read))
{
destinationBlob.UploadFromStream(fs, null, null, null);
}
#endif
}
/// <summary>
/// read bytes from the stream and append the content to an existed file
/// </summary>
/// <param name="blobPath"></param>
/// <param name="stream"></param>
/// <returns></returns>
public async Task <bool> AppendFileFromStream(string path, Stream stream)
{
var blobPath = path.ToAzurePath();
var container = _container;
rootFix(ref container, ref blobPath);
CloudBlob rawBlob = container.GetBlobReference(blobPath);
if (rawBlob.Properties.BlobType == BlobType.PageBlob)
{
return(false); //for page blob, append is not supported
}
CloudBlockBlob blob = container.GetBlockBlobReference(blobPath);
// store the block id of this blob
var blockList = new List <string>();
try
{
foreach (var block in await blob.DownloadBlockListAsync())
{
blockList.Add(block.Name);
}
}
catch (StorageException)
{
// do nothing, this may happen when blob doesn't exist
}
const int blockSize = 4 * 1024 * 1024; // 4M - block size
byte[] buffer = new byte[blockSize];
// append file
try
{
int nRead = 0;
while (nRead < blockSize)
{
int actualRead = stream.Read(buffer, nRead, blockSize - nRead);
if (actualRead <= 0) // stream end
{
//put last block & break
string strBlockId = GetBlockID(blockList);
await blob.PutBlockAsync(strBlockId, new System.IO.MemoryStream(buffer, 0, nRead), null);
blockList.Add(strBlockId);
break;
}
else if (actualRead == (blockSize - nRead))// buffer full
{
//put this block
string strBlockId = GetBlockID(blockList);
await blob.PutBlockAsync(strBlockId, new System.IO.MemoryStream(buffer), null);
blockList.Add(strBlockId);
nRead = 0;
continue;
}
nRead += actualRead;
}
var ext = Path.GetExtension(path);
blob.Properties.ContentType = MimeTypes.Core.MimeTypeMap.GetMimeType(ext);
await blob.SetPropertiesAsync();
}
catch (StorageException)
{
// blob.PutBlock error
return(false);
}
// put block list
await blob.PutBlockListAsync(blockList);
return(true);
}
/// <summary>
/// Appends a stream's contents to the end of a blob (creating the blob and appending a header if the blob doesn't exist).
/// Before calling this method, set the blob's ContentType Property (ex: "text/csv; charset=utf-8")
/// </summary>
/// <param name="blob">The blob to have the byte array appended to.</param>
/// <param name="data">The stream with contents to append.</param>
/// <param name="header">The header byte array to put in the blob if the blob is being created.</param>
/// <param name="maxBlockSize">The maximum block sized in bytes (0=Azure default of 4MB)</param>
/// <param name="options">A Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions object that specifies additional options for the request.</param>
/// <param name="operationContext">An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation.</param>
/// <param name="cancellationToken">A System.Threading.CancellationToken to observe while waiting for a task to complete.</param>
/// <returns>A System.Threading.Tasks.Task object that represents the current operation.</returns>
public static async Task AppendAsync(this CloudBlockBlob blob, Stream data, Byte[] header, Int32 maxBlockSize = 0,
BlobRequestOptions options = null, OperationContext operationContext = null,
CancellationToken cancellationToken = default(CancellationToken))
{
if (maxBlockSize == 0)
{
maxBlockSize = 4 * 1024 * 1024;
}
if (data.Length > maxBlockSize)
{
throw new ArgumentOutOfRangeException("data", "A single data object cannot be larger than " + maxBlockSize.ToString() + " bytes.");
}
if (header != null && header.Length > maxBlockSize)
{
throw new ArgumentOutOfRangeException("header", "The header cannot be larger than " + maxBlockSize.ToString() + " bytes.");
}
while (true)
{
using (var ms = new MemoryStream()) {
AccessCondition accessCondition;
IEnumerable <ListBlockItem> blockList = Enumerable.Empty <ListBlockItem>();
try {
blockList = await blob.DownloadBlockListAsync(BlockListingFilter.Committed,
null, options, operationContext, cancellationToken).ConfigureAwait(false); // 404 if blob not found
accessCondition = AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag); // Write if blob matches what we read
}
catch (StorageException se) {
if (!se.Matches(HttpStatusCode.NotFound, BlobErrorCodeStrings.BlobNotFound))
{
throw;
}
accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); // Write if blob doesn't exist
}
try {
List <String> blockIds = blockList.Select(lbi => lbi.Name).ToList();
ListBlockItem lastBlock = blockList.LastOrDefault();
if (lastBlock == null)
{
if (header != null)
{
// No blocks exist, add header (if specified)
ms.Write(header, 0, header.Length);
}
}
else
{
// A block exists, can it hold the new data?
if (lastBlock.Length + data.Length < maxBlockSize)
{
// Yes, download the last block's current data as long as the blob's etag hasn't changed
Int64 lastBlockOffset = blockList.Sum(lbi => lbi.Length) - lastBlock.Length;
await blob.DownloadRangeToStreamAsync(ms, lastBlockOffset, lastBlock.Length,
accessCondition, options, operationContext, cancellationToken).ConfigureAwait(false); // 412 if blob modified behind our back
blockIds.Remove(lastBlock.Name); // Remove the block we're appending to
}
}
await data.CopyToAsync(ms).ConfigureAwait(false); // Append new data to end of stream
ms.Seek(0, SeekOrigin.Begin);
// Upload new block and append it to the blob
String newBlockId = Guid.NewGuid().ToString("N").Encode().ToBase64String();
blockIds.Add(newBlockId); // Append new block to end of blob
await blob.PutBlockAsync(newBlockId, ms, null, accessCondition,
options, operationContext, cancellationToken).ConfigureAwait(false); // PutBlock ignores access condition so this always succeeds
await blob.PutBlockListAsync(blockIds, accessCondition,
options, operationContext, cancellationToken).ConfigureAwait(false); // 409 if blob created behind our back; 400 if block ID doesn't exist (happens in another PC calls PutBlockList after our PutBlock)
break; // If successful, we're done; don't retry
}
catch (StorageException se) {
// Blob got created behind our back, retry
if (se.Matches(HttpStatusCode.Conflict, BlobErrorCodeStrings.BlobAlreadyExists))
{
continue;
}
// Blob got created or modified behind our back, retry
if (se.Matches(HttpStatusCode.PreconditionFailed))
{
continue;
}
// Another PC called PutBlockList between our PutBlock & PutBlockList,
// our block got destroyed, retry
if (se.Matches(HttpStatusCode.BadRequest, BlobErrorCodeStrings.InvalidBlockList))
{
continue;
}
throw;
}
}
}
}
