private async static void UpdateCertainBlock() { try { CloudBlobContainer container = _cloudBlobClient.GetContainerReference("text-files"); if (await container.ExistsAsync()) { CloudBlockBlob fileBlob = container.GetBlockBlobReference("sample.txt"); if (await fileBlob.ExistsAsync()) { List <ListBlockItem> ids = new List <ListBlockItem>(await fileBlob.DownloadBlockListAsync()); if (ids.Count > 0) { string blockId = "0x002"; string blockIdBase64 = Convert.ToBase64String(UTF8Encoding.UTF8.GetBytes(blockId)); string newContent = "000000000" + Environment.NewLine; List <string> idList = new List <string>(); if (ids.Any(x => x.Name.Equals(blockIdBase64, StringComparison.InvariantCultureIgnoreCase))) { await fileBlob.PutBlockAsync(blockIdBase64, new MemoryStream(UTF8Encoding.UTF8.GetBytes(newContent), true), null); idList.AddRange(ids.Select(x => x.Name)); } else { await fileBlob.PutBlockAsync(blockIdBase64, new MemoryStream(UTF8Encoding.UTF8.GetBytes(newContent), true), null); idList.AddRange(ids.Select(x => x.Name)); idList.Add(blockIdBase64); Console.WriteLine("Target block is not found. Additional Block has been added"); } await fileBlob.PutBlockListAsync(idList); Console.WriteLine("Done"); } else { Console.WriteLine("No available blocks in the file"); } } else { Console.WriteLine("File doesn't exist"); } } else { Console.WriteLine("Container doesn't exist"); } } catch (Exception ex) { Console.WriteLine(ex); } }
private static async Task AppendCommitInternalAsync(CloudBlockBlob blob, string[] blockIds, CancellationToken cancellationToken) { IEnumerable <ListBlockItem> blockList = await blob.DownloadBlockListAsync( BlockListingFilter.Committed, accessCondition : null, options : null, operationContext : null, cancellationToken); List <string> newBlockLists = blockList.Select(b => b.Name).ToList(); newBlockLists.AddRange(blockIds); await CommitInternalAsync(blob, newBlockLists.ToArray(), cancellationToken); }
/// <summary> /// Reads the blob's block list, and indicates whether the blob has been committed. /// </summary> /// <param name="blob">A CloudBlockBlob object.</param> /// <returns>A Task object.</returns> private static async Task ReadBlockListAsync(CloudBlockBlob blob) { // Get the blob's block list. foreach (var listBlockItem in await blob.DownloadBlockListAsync(BlockListingFilter.All, null, null, null)) { Console.WriteLine( listBlockItem.Committed ? "Block {0} has been committed to block list. Block length = {1}" : "Block {0} is uncommitted. Block length = {1}", listBlockItem.Name, listBlockItem.Length); } Console.WriteLine(); }
/// <summary> /// Waits for all expected blocks to finish uploading, commits the blob, then deletes the blob and container used during the test. /// </summary> /// <remarks>This method will only be called if application is running as ID 0.</remarks> /// <param name="container">The blob's container.</param> /// <param name="blobName">The name of the blob being uploaded.</param> /// <param name="totalBlocks">The total number of blocks to commit to the blob.</param> private static async Task FinalizeBlob(CloudBlobContainer container, string blobName, uint totalBlocks, bool cleanup) { // Define the order in which to commit blocks. List <string> blockIdList = new List <string>(); for (uint i = 0; i < totalBlocks; ++i) { blockIdList.Add(Convert.ToBase64String(BitConverter.GetBytes(i))); } HashSet <string> blockIdSet = new HashSet <string>(blockIdList); CloudBlockBlob blockblob = container.GetBlockBlobReference(blobName); // Poll the blob's Uncommitted Block List until we determine that all expected blocks have been successfully uploaded. Console.WriteLine("Waiting for all expected blocks to appear in the uncommitted block list."); IEnumerable <ListBlockItem> currentUncommittedBlockList = new List <ListBlockItem>(); while (true) { currentUncommittedBlockList = await blockblob.DownloadBlockListAsync(BlockListingFilter.Uncommitted, AccessCondition.GenerateEmptyCondition(), null, null); if (currentUncommittedBlockList.Count() >= blockIdList.Count && VerifyBlocks(currentUncommittedBlockList, blockIdSet)) { break; } Console.WriteLine($"{currentUncommittedBlockList.Count()} / {blockIdList.Count} blocks in the uncommitted block list."); await Task.Delay(TimeSpan.FromSeconds(5)); } Console.WriteLine("Issuing PutBlockList."); await blockblob.PutBlockListAsync(blockIdList); if (cleanup) { Console.WriteLine("Press 'Enter' key to delete the example container and blob."); Console.ReadLine(); Console.WriteLine("Issuing DeleteBlob."); await blockblob.DeleteAsync(); Console.WriteLine("Deleting container."); await container.DeleteAsync(); } }
public async Task <int> GetBlockListAsync() { StorageCredentials creds = new StorageCredentials(AcctName, AcctKey); CloudStorageAccount account = new CloudStorageAccount(creds, useHttps: true); CloudBlobClient client = account.CreateCloudBlobClient(); CloudBlockBlob blob = new CloudBlockBlob(Uri, client); System.Collections.Generic.IEnumerable <ListBlockItem> list = await blob.DownloadBlockListAsync(); int count = 0; foreach (var item in list) { count++; } return(count); }
public void UploadBlock(Stream blockStream, string containerName, string fileName, int blockIndex, int blocksCount, string contentType) { string blockId = Convert.ToBase64String(BitConverter.GetBytes(blockIndex)); CloudBlockBlob blob = GetBlockBlob(containerName, null, fileName); blob.PutBlockAsync(blockId, blockStream, null).Wait(); if (blockIndex == blocksCount - 1) { BlockListingFilter blockType = BlockListingFilter.Uncommitted; IEnumerable <ListBlockItem> blockList = blob.DownloadBlockListAsync(blockType, null, null, null).Result; List <string> blockIds = new List <string>(); foreach (ListBlockItem blockItem in blockList) { blockIds.Add(blockItem.Name); } blob.PutBlockListAsync(blockIds).Wait(); blob.Properties.ContentType = contentType; blob.SetPropertiesAsync().Wait(); } }
internal async static Task <IEnumerable <ListBlockItem> > GetBlockListForBlob(CloudBlockBlob destBlob, int retryCount, ILogger logger) { IEnumerable <ListBlockItem> retVal = null; // use retry policy which will automatically handle the throttling related StorageExceptions await GetStorageRetryPolicy($"DownloadBlockListAsync for blob {destBlob.Name}", retryCount, logger).ExecuteAsync(async() => { if (!await destBlob.ExistsAsync()) { // obvious but setting explicitly retVal = null; } else { retVal = await destBlob.DownloadBlockListAsync(BlockListingFilter.Committed, null, null, null); } }); return(retVal); }
/// <summary> /// Writes data from a stream to a blob. /// </summary> /// <param name="fileStream">The stream to read from.</param> /// <param name="connectionString">The connection string for the destination blob storage.</param> /// <param name="containerName">The name of the container to place the blob.</param> /// <param name="destinationPath">The relative path (from the container) for the blob.</param> /// <param name="createContainerIfNotExists">Specifies if the container should be created if it does not exist.</param> /// <returns>A <see cref="Microsoft.AdvocacyPlatform.Contracts.FileInfo"/> object representing the new blob.</returns> public async Task <Contracts.FileInfo> WriteStreamAsync(Stream fileStream, Secret connectionString, string containerName, string destinationPath, bool createContainerIfNotExists) { try { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString.Value); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(containerName); if (createContainerIfNotExists) { await container.CreateIfNotExistsAsync(); } CloudBlockBlob blockBlob = container.GetBlockBlobReference(destinationPath); bool exists = await blockBlob.ExistsAsync(); if (exists) { throw new RecordingExistsException(destinationPath); } await blockBlob.UploadFromStreamAsync(fileStream); IEnumerable <ListBlockItem> blockBlobBlocks = await blockBlob.DownloadBlockListAsync(); return(new Contracts.FileInfo() { SizeInBytes = blockBlobBlocks.Sum(x => x.Length), Uri = blockBlob.Uri, }); } catch (StorageException ex) { throw new StorageClientException("Storage client encountered exception", ex); } }
// // Function for calculating the size of a blob // private static void CalculateBlobSize(string accountName, object containerName, object blobName, string sasToken, bool isDisk) { // Now that we hvae the SAS-token, let's calculate the size of the snapshot var blobSizeInBytes = 0F; System.Console.WriteLine("Trying to retrieve Blob..."); var blob = default(ICloudBlob); var blobUri = new Uri($"https://{accountName}.blob.core.windows.net/{containerName}/{blobName}?{sasToken}"); try { var reqOptions = new BlobRequestOptions { ServerTimeout = TimeSpan.FromMinutes(5), MaximumExecutionTime = TimeSpan.FromMinutes(5) }; if (isDisk) { var pageBlob = new CloudPageBlob(blobUri); blob = pageBlob; Task.WaitAll(blob.FetchAttributesAsync()); var blobSize = pageBlob.Properties.Length; System.Console.WriteLine($"Blob size per API in bytes: {blobSize}."); System.Console.WriteLine($"Blob size per API in MB: {blobSize / 1024 / 1024}."); System.Console.WriteLine($"Blob size per API in GB: {blobSize / 1024 / 1024 / 1024}."); var pageRanges = pageBlob.GetPageRangesAsync().Result; foreach (var rg in pageRanges) { blobSizeInBytes += (12 + (rg.EndOffset - rg.StartOffset)); } } else { var blockBlob = new CloudBlockBlob(blobUri); blob = blockBlob; blobSizeInBytes += 8; var blockList = blockBlob.DownloadBlockListAsync().Result; foreach (var bl in blockList) { blobSizeInBytes += bl.Length + bl.Name.Length; } } } catch (Exception ex) { System.Console.WriteLine($"EXCPETION: {ex.Message}"); if (ex.InnerException != null) { System.Console.WriteLine($"INNER EXCEPTION: {ex.InnerException.Message}"); } } // Add the base size to the previously calculated size blobSizeInBytes += (124 + (blob.Name.Length * 2)); foreach (var md in blob.Metadata) { blobSizeInBytes += (3 + (md.Key.Length + md.Value.Length)); } // Output the final size of the blob System.Console.WriteLine(""); System.Console.WriteLine($"Blob Size per calculation in bytes: {blobSizeInBytes}"); System.Console.WriteLine($"Blob Size per calculation in MB: {blobSizeInBytes / 1024 / 1024}"); System.Console.WriteLine($"Blob Size per calculation in GB: {blobSizeInBytes / 1024 / 1024 / 1024}"); }
static void Main(string[] args) { var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json"); Configuration = builder.Build(); CloudStorageAccount account = CloudStorageAccount.Parse(Configuration["AppSettings:storageConnection"]); CloudQueueClient queueClient = account.CreateCloudQueueClient(); CloudQueue queue = queueClient.GetQueueReference("blobcreateevents"); queue.CreateIfNotExistsAsync(); CloudBlobClient blobClient = account.CreateCloudBlobClient(); bool done = false; ListBlockItem lastBlockDownloaded = null; int currentBackOff = 0; int maxBackOff = 5; byte[] bytesFromBlob = new byte[4 * 1024 * 1024]; do { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("Getting messages from the queue."); queue.FetchAttributesAsync().Wait(); int countOfMessages = Convert.ToInt32(queue.ApproximateMessageCount); if (countOfMessages > 0) { currentBackOff = 1; var msgs = queue.GetMessagesAsync(countOfMessages > 32 ? 32 : countOfMessages).Result; var numberOfMessagesLeftInBatch = msgs.Count(); foreach (var msg in msgs) { if (numberOfMessagesLeftInBatch == 1) // process only the last message // all messages are equivalent in meaning // sender usually gets ahead of receiver { JToken token = JObject.Parse(msg.AsString); string resourceId = (string)token.SelectToken("topic"); string subject = (string)token.SelectToken("subject"); string eventType = (string)token.SelectToken("eventType"); if (eventType == "Microsoft.Storage.BlobCreated") { var storageAccountName = resourceId.Split('/')[8]; var storageContainerName = subject.Split('/')[4]; var blobName = subject.Split('/')[6]; Uri uri = new Uri($"https://{storageAccountName}.blob.core.windows.net/{storageContainerName}/{blobName}"); CloudBlockBlob blob = new CloudBlockBlob(uri, blobClient); var blockList = blob.DownloadBlockListAsync().Result; long blobOffset = 0; var blocksToCopy = blockList; if (lastBlockDownloaded != null) { // count the blocks already written var countOfblocksAlreadyWritten = blockList.TakeWhile(item => (item.Name != lastBlockDownloaded.Name)).Count() + 1; // get an enumerable of those block list items var blocksAlreadyWritten = blockList.Take(countOfblocksAlreadyWritten); // add up the bytes already written foreach (var block in blocksAlreadyWritten) { blobOffset += block.Length; } // skip over blocks already written blocksToCopy = blockList.SkipWhile(item => (item.Name != lastBlockDownloaded.Name)).Skip(1); } if (blocksToCopy.Count() > 0) { var fs = File.OpenWrite(@"c:\temp\abigfile.dat"); using (BinaryWriter writer = new BinaryWriter(fs)) { foreach (var block in blocksToCopy) { blob.DownloadRangeToByteArrayAsync(bytesFromBlob, 0, blobOffset, block.Length).Wait(); writer.Seek(0, SeekOrigin.End); writer.Write(bytesFromBlob, 0, (int)block.Length); blobOffset += block.Length; Console.ForegroundColor = ConsoleColor.White; Console.WriteLine($"{block.Name}"); } }; fs.Close(); } lastBlockDownloaded = blockList.Last(); } } Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("Deleting a message from the queue"); queue.DeleteMessageAsync(msg); numberOfMessagesLeftInBatch--; } } else { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine($"Waiting for {currentBackOff} seconds for next message"); Thread.Sleep(currentBackOff * 1000); currentBackOff += (currentBackOff < maxBackOff ? 1 : 0); } // set done flag in some way, or loop forever. } while (!done); Console.ReadKey(); }
public static async Task <string> UploadAsync(Func <long, int, Task <byte[]> > fetchLocalData, long blobLenth, CloudBlockBlob blockBlob, uint uploadParallelism = DEFAULT_PARALLELISM) { const int MAXIMUM_UPLOAD_SIZE = 4 * MB; if (NumBytesPerChunk > MAXIMUM_UPLOAD_SIZE) { NumBytesPerChunk = MAXIMUM_UPLOAD_SIZE; } #region Which blocks exist in the file var allBlockInFile = Enumerable .Range(0, 1 + ((int)(blobLenth / NumBytesPerChunk))) .Select(_ => new BlockMetadata(_, blobLenth, NumBytesPerChunk)) .Where(block => block.Length > 0) .ToList(); var blockIdList = allBlockInFile.Select(_ => _.BlockId).ToList(); #endregion #region Which blocks are already uploaded List <BlockMetadata> missingBlocks = null; try { var existingBlocks = (await blockBlob.DownloadBlockListAsync( BlockListingFilter.Uncommitted, AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions { }, new OperationContext { })) .Where(_ => _.Length == NumBytesPerChunk) .ToList(); missingBlocks = allBlockInFile.Where(blockInFile => !existingBlocks.Any(existingBlock => existingBlock.Name == blockInFile.BlockId && existingBlock.Length == blockInFile.Length)).ToList(); } catch (StorageException) { missingBlocks = allBlockInFile; } #endregion Func <BlockMetadata, Statistics, Task> uploadBlockAsync = async(block, stats) => { byte[] blockData = await fetchLocalData(block.Index, block.Length); string contentHash = md5()(blockData); DateTime start = DateTime.UtcNow; await ExecuteUntilSuccessAsync(async() => { await blockBlob.PutBlockAsync( blockId: block.BlockId, blockData: new MemoryStream(blockData, true), contentMD5: contentHash, accessCondition: AccessCondition.GenerateEmptyCondition(), options: new BlobRequestOptions { StoreBlobContentMD5 = true, UseTransactionalMD5 = true }, operationContext: new OperationContext()); }, consoleExceptionHandler); stats.Add(block.Length, start); }; var s = new Statistics(missingBlocks.Sum(b => (long)b.Length)); await LargeFileUploaderUtils.ForEachAsync( source : missingBlocks, parallelUploads : 4, body : blockMetadata => uploadBlockAsync(blockMetadata, s)); await ExecuteUntilSuccessAsync(async() => { await blockBlob.PutBlockListAsync(blockIdList); }, consoleExceptionHandler); log("PutBlockList succeeded, finished upload to {0}", blockBlob.Uri.AbsoluteUri); return(blockBlob.Uri.AbsoluteUri); }
public static IEnumerable <ListBlockItem> DownloadBlockList(this CloudBlockBlob blob, BlockListingFilter blockListingFilter = BlockListingFilter.Committed, AccessCondition accessCondition = null, BlobRequestOptions options = null, OperationContext operationContext = null) { return(blob.DownloadBlockListAsync(blockListingFilter, accessCondition, options, operationContext).GetAwaiter().GetResult()); }
/// <summary>Appends a block to the end of a block blob.</summary> /// <param name="blob">The blob to append a block to.</param> /// <param name="info">Additional info used when the blob is first being created.</param> /// <param name="dataToAppend">The data to append as a block to the end of the blob.</param> /// <param name="options">The blob request options.</param> /// <param name="operationContext">The operation context.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>A Task indicating when the operation has completed.</returns> public static async Task AppendBlockAsync(this CloudBlockBlob blob, AppendBlobBlockInfo info, Byte[] dataToAppend, BlobRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { blob.Properties.ContentType = info.MimeType; //blob.Properties.ContentDisposition = "attachment; filename=\"fname.ext\""; // NOTE: Put Block ignores access condition and so it always succeeds List <String> blockList = new List <String>(); while (true) { blockList.Clear(); AccessCondition accessCondition; try { blockList.AddRange((await blob.DownloadBlockListAsync(BlockListingFilter.Committed, null, options, operationContext, cancellationToken).ConfigureAwait(false)).Select(lbi => lbi.Name)); // 404 if blob not found accessCondition = AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag); // Write if blob matches what we read } catch (StorageException se) { if (!se.Matches(HttpStatusCode.NotFound, BlobErrorCodeStrings.BlobNotFound)) { throw; } accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); // Write if blob doesn't exist } try { if (blockList.Count == 0) // Blob doesn't exist yet, add header (if specified) // Notify client code that new blob is about to be created { info.OnCreatingBlob(blob); if (info.Header != null) // Write header to new blob { String headerBlockId = Guid.NewGuid().ToString().Encode().ToBase64String(); await blob.PutBlockAsync(headerBlockId, new MemoryStream(info.Header), null).ConfigureAwait(false); blockList.Add(headerBlockId); } } // Upload new block & add it's Id to the block list String blockId = Guid.NewGuid().ToString().Encode().ToBase64String(); await blob.PutBlockAsync(blockId, new MemoryStream(dataToAppend), null).ConfigureAwait(false); blockList.Add(blockId); // If too many blocks, remove old block (but not the header if it exists) var maxEntries = info.MaxEntries + ((info.Header == null) ? 0 : 1); if (blockList.Count > maxEntries) { blockList.RemoveAt((info.Header == null) ? 0 : 1); } // Upload the new block list await blob.PutBlockListAsync(blockList, AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag), options, operationContext, cancellationToken).ConfigureAwait(false); // 409 if blob created behind our back; 400 if block Id doesn't exist (happens in another PC calls PutBlockList after our PutBlock) break; // If successful, we're done; don't retry } catch (StorageException se) { // Blob got created behind our back, retry if (se.Matches(HttpStatusCode.Conflict, BlobErrorCodeStrings.BlobAlreadyExists)) { continue; } // Blob got created or modified behind our back, retry if (se.Matches(HttpStatusCode.PreconditionFailed)) { continue; } // Another PC called PutBlockList between our PutBlock & PutBlockList, // our block(s) got destroyed, retry if (se.Matches(HttpStatusCode.BadRequest, BlobErrorCodeStrings.InvalidBlockList)) { continue; } throw; } } }
public static async Task <string> UploadAsync(Func <long, int, Task <byte[]> > fetchLocalData, long blobLenth, CloudBlockBlob blockBlob, uint uploadParallelism = DEFAULT_PARALLELISM) { const int MAXIMUM_UPLOAD_SIZE = 100 * MB; if (NumBytesPerChunk > MAXIMUM_UPLOAD_SIZE) { NumBytesPerChunk = MAXIMUM_UPLOAD_SIZE; } #region Which blocks exist in the file var allBlockInFile = Enumerable .Range(0, 1 + ((int)(blobLenth / NumBytesPerChunk))) .Select(_ => new BlockMetadata(_, blobLenth, NumBytesPerChunk)) .Where(block => block.Length > 0) .ToList(); var blockIdList = allBlockInFile.Select(_ => _.BlockId).ToList(); #endregion #region Which blocks are already uploaded List <BlockMetadata> missingBlocks = null; try { var existingBlocks = (await blockBlob.DownloadBlockListAsync( BlockListingFilter.Uncommitted, AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions { }, new OperationContext { })) .Where(_ => _.Length == NumBytesPerChunk) .ToList(); missingBlocks = allBlockInFile.Where(blockInFile => !existingBlocks.Any(existingBlock => existingBlock.Name == blockInFile.BlockId && existingBlock.Length == blockInFile.Length)).ToList(); } catch (StorageException) { missingBlocks = allBlockInFile; } #endregion Func <BlockMetadata, Statistics, Task> uploadBlockAsync = async(block, stats) => { DateTime start = DateTime.UtcNow; stats.Add(block.Length, start); var fetchLocalDataResult = await Policy .Handle <Exception>() .WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt))) .ExecuteAndCaptureAsync <byte[]>(() => fetchLocalData(block.Index, block.Length)); if (fetchLocalDataResult.Outcome == OutcomeType.Failure) { throw new Exception($"Could not read local file", fetchLocalDataResult.FinalException); } byte[] blockData = fetchLocalDataResult.Result; string contentHash = md5()(blockData); var putBlockAsyncResult = await Policy .Handle <Exception>() .WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt))) .ExecuteAndCaptureAsync(() => blockBlob.PutBlockAsync( blockId: block.BlockId, blockData: new MemoryStream(blockData, true), contentMD5: contentHash, accessCondition: AccessCondition.GenerateEmptyCondition(), options: new BlobRequestOptions { StoreBlobContentMD5 = true, UseTransactionalMD5 = true }, operationContext: new OperationContext())); if (putBlockAsyncResult.Outcome == OutcomeType.Failure) { throw new Exception($"Could not call PutBlockAsync {putBlockAsyncResult.FinalException.Message}", putBlockAsyncResult.FinalException); } }; var s = new Statistics(missingBlocks.Sum(b => (long)b.Length)); await LargeFileUploaderUtils.ForEachAsync( source : missingBlocks, parallelTasks : 4, task : blockMetadata => uploadBlockAsync(blockMetadata, s)); var putBlockListAsyncResult = await Policy .Handle <Exception>() .WaitAndRetryAsync(retryCount: 5, sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt))) .ExecuteAndCaptureAsync(() => blockBlob.PutBlockListAsync(blockIdList)); if (putBlockListAsyncResult.Outcome == OutcomeType.Failure) { throw new Exception($"Could not call PutBlockListAsync", putBlockListAsyncResult.FinalException); } log("PutBlockList succeeded, finished upload to {0}", blockBlob.Uri.AbsoluteUri); return(blockBlob.Uri.AbsoluteUri); }
public static async Task Run( [BlobTrigger("%blobContainerName%/resourceId=/SUBSCRIPTIONS/{subId}/RESOURCEGROUPS/{resourceGroup}/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/{nsgName}/y={blobYear}/m={blobMonth}/d={blobDay}/h={blobHour}/m={blobMinute}/macAddress={mac}/PT1H.json", Connection = "nsgSourceDataConnection")] CloudBlockBlob myBlob, [Table("checkpoints", Connection = "AzureWebJobsStorage")] CloudTable checkpointTable, Binder nsgDataBlobBinder, Binder cefLogBinder, string subId, string resourceGroup, string nsgName, string blobYear, string blobMonth, string blobDay, string blobHour, string blobMinute, string mac, ExecutionContext executionContext, ILogger log) { log.LogDebug($"BlobTriggerIngestAndTransmit triggered: {executionContext.InvocationId} "); string nsgSourceDataAccount = Util.GetEnvironmentVariable("nsgSourceDataAccount"); if (nsgSourceDataAccount.Length == 0) { log.LogError("Value for nsgSourceDataAccount is required."); throw new System.ArgumentNullException("nsgSourceDataAccount", "Please provide setting."); } string blobContainerName = Util.GetEnvironmentVariable("blobContainerName"); if (blobContainerName.Length == 0) { log.LogError("Value for blobContainerName is required."); throw new System.ArgumentNullException("blobContainerName", "Please provide setting."); } string outputBinding = Util.GetEnvironmentVariable("outputBinding"); if (outputBinding.Length == 0) { log.LogError("Value for outputBinding is required. Permitted values are: 'arcsight', 'splunk', 'eventhub'."); throw new System.ArgumentNullException("outputBinding", "Please provide setting."); } var blobDetails = new BlobDetails(subId, resourceGroup, nsgName, blobYear, blobMonth, blobDay, blobHour, blobMinute, mac); // get checkpoint Checkpoint checkpoint = Checkpoint.GetCheckpoint(blobDetails, checkpointTable); var blockList = myBlob.DownloadBlockListAsync().Result; var startingByte = blockList.Where((item, index) => index < checkpoint.CheckpointIndex).Sum(item => item.Length); var endingByte = blockList.Where((item, index) => index < blockList.Count() - 1).Sum(item => item.Length); var dataLength = endingByte - startingByte; log.LogDebug("Blob: {0}, starting byte: {1}, ending byte: {2}, number of bytes: {3}", blobDetails.ToString(), startingByte, endingByte, dataLength); if (dataLength == 0) { log.LogWarning(string.Format("Blob: {0}, triggered on completed hour.", blobDetails.ToString())); return; } //foreach (var item in blockList) //{ // log.LogInformation("Name: {0}, Length: {1}", item.Name, item.Length); //} var attributes = new Attribute[] { new BlobAttribute(string.Format("{0}/{1}", blobContainerName, myBlob.Name)), new StorageAccountAttribute(nsgSourceDataAccount) }; string nsgMessagesString = ""; var bytePool = ArrayPool <byte> .Shared; byte[] nsgMessages = bytePool.Rent((int)dataLength); try { CloudBlockBlob blob = nsgDataBlobBinder.BindAsync <CloudBlockBlob>(attributes).Result; await blob.DownloadRangeToByteArrayAsync(nsgMessages, 0, startingByte, dataLength); if (nsgMessages[0] == ',') { nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages, 1, (int)(dataLength - 1)); } else { nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages, 0, (int)dataLength); } } catch (Exception ex) { log.LogError(string.Format("Error binding blob input: {0}", ex.Message)); throw ex; } finally { bytePool.Return(nsgMessages); } //log.LogDebug(nsgMessagesString); try { int bytesSent = await Util.SendMessagesDownstreamAsync(nsgMessagesString, executionContext, cefLogBinder, log); log.LogInformation($"Sending {nsgMessagesString.Length} bytes (denormalized to {bytesSent} bytes) downstream via output binding {outputBinding}."); } catch (Exception ex) { log.LogError(string.Format("SendMessagesDownstreamAsync: Error {0}", ex.Message)); throw ex; } checkpoint.PutCheckpoint(checkpointTable, blockList.Count() - 1); }
private static void CreateStreamAndUploadToBlob(string sourceFile, CloudBlockBlob destinationBlob) { #if DotNetCoreClr if (destinationBlob.ExistsAsync().Result&& destinationBlob.DownloadBlockListAsync().Result.Any()) { // Delete all blocks in case we crashed and failed the upload in previous attempt. destinationBlob.PutBlockListAsync(new List <string>()).Wait(); } int blockSize = 4 * 1024 * 1024; //4 MB using (FileStream fileStream = new FileStream(sourceFile, FileMode.Open, FileAccess.Read)) { var tasks = new List <Task>(); long fileSize = fileStream.Length; //block count is the number of blocks + 1 for the last one int blockCount = (int)((float)fileSize / (float)blockSize) + 1; //List of block ids; the blocks will be committed in the order of this list List <string> blockIDs = new List <string>(); //starting block number int blockNumber = 0; int bytesRead = 0; //number of bytes read so far long bytesLeft = fileSize; //number of bytes left to read and upload //do until all of the bytes are uploaded while (bytesLeft > 0) { blockNumber++; int bytesToRead; if (bytesLeft >= blockSize) { //more than one block left, so put up another whole block bytesToRead = blockSize; } else { //less than one block left, read the rest of it bytesToRead = (int)bytesLeft; } //create a blockID from the block number, add it to the block ID list //the block ID is a base64 string string blockId = Convert.ToBase64String(System.Text.ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}", blockNumber.ToString("d7")))); blockIDs.Add(blockId); //set up new buffer with the right size, and read that many bytes into it byte[] bytes = new byte[bytesToRead]; fileStream.Read(bytes, 0, bytesToRead); //calculate the MD5 hash of the byte array string blockHash = GetMD5HashFromStream(bytes); //upload the block, provide the hash so Azure can verify it tasks.Add(destinationBlob.PutBlockAsync(blockId, new MemoryStream(bytes), blockHash)); //increment/decrement counters bytesRead += bytesToRead; bytesLeft -= bytesToRead; // Do not start more instances to limit memory usage if (tasks.Count == 5) { Task.WaitAll(tasks.ToArray()); tasks.Clear(); } } if (tasks.Any()) { Task.WaitAll(tasks.ToArray()); } //commit the blocks destinationBlob.PutBlockListAsync(blockIDs).Wait(); } #else using (FileStream fs = new FileStream(sourceFile, FileMode.Open, FileAccess.Read)) { destinationBlob.UploadFromStream(fs, null, null, null); } #endif }
/// <summary> /// read bytes from the stream and append the content to an existed file /// </summary> /// <param name="blobPath"></param> /// <param name="stream"></param> /// <returns></returns> public async Task <bool> AppendFileFromStream(string path, Stream stream) { var blobPath = path.ToAzurePath(); var container = _container; rootFix(ref container, ref blobPath); CloudBlob rawBlob = container.GetBlobReference(blobPath); if (rawBlob.Properties.BlobType == BlobType.PageBlob) { return(false); //for page blob, append is not supported } CloudBlockBlob blob = container.GetBlockBlobReference(blobPath); // store the block id of this blob var blockList = new List <string>(); try { foreach (var block in await blob.DownloadBlockListAsync()) { blockList.Add(block.Name); } } catch (StorageException) { // do nothing, this may happen when blob doesn't exist } const int blockSize = 4 * 1024 * 1024; // 4M - block size byte[] buffer = new byte[blockSize]; // append file try { int nRead = 0; while (nRead < blockSize) { int actualRead = stream.Read(buffer, nRead, blockSize - nRead); if (actualRead <= 0) // stream end { //put last block & break string strBlockId = GetBlockID(blockList); await blob.PutBlockAsync(strBlockId, new System.IO.MemoryStream(buffer, 0, nRead), null); blockList.Add(strBlockId); break; } else if (actualRead == (blockSize - nRead))// buffer full { //put this block string strBlockId = GetBlockID(blockList); await blob.PutBlockAsync(strBlockId, new System.IO.MemoryStream(buffer), null); blockList.Add(strBlockId); nRead = 0; continue; } nRead += actualRead; } var ext = Path.GetExtension(path); blob.Properties.ContentType = MimeTypes.Core.MimeTypeMap.GetMimeType(ext); await blob.SetPropertiesAsync(); } catch (StorageException) { // blob.PutBlock error return(false); } // put block list await blob.PutBlockListAsync(blockList); return(true); }
/// <summary> /// Appends a stream's contents to the end of a blob (creating the blob and appending a header if the blob doesn't exist). /// Before calling this method, set the blob's ContentType Property (ex: "text/csv; charset=utf-8") /// </summary> /// <param name="blob">The blob to have the byte array appended to.</param> /// <param name="data">The stream with contents to append.</param> /// <param name="header">The header byte array to put in the blob if the blob is being created.</param> /// <param name="maxBlockSize">The maximum block sized in bytes (0=Azure default of 4MB)</param> /// <param name="options">A Microsoft.WindowsAzure.Storage.Blob.BlobRequestOptions object that specifies additional options for the request.</param> /// <param name="operationContext">An Microsoft.WindowsAzure.Storage.OperationContext object that represents the context for the current operation.</param> /// <param name="cancellationToken">A System.Threading.CancellationToken to observe while waiting for a task to complete.</param> /// <returns>A System.Threading.Tasks.Task object that represents the current operation.</returns> public static async Task AppendAsync(this CloudBlockBlob blob, Stream data, Byte[] header, Int32 maxBlockSize = 0, BlobRequestOptions options = null, OperationContext operationContext = null, CancellationToken cancellationToken = default(CancellationToken)) { if (maxBlockSize == 0) { maxBlockSize = 4 * 1024 * 1024; } if (data.Length > maxBlockSize) { throw new ArgumentOutOfRangeException("data", "A single data object cannot be larger than " + maxBlockSize.ToString() + " bytes."); } if (header != null && header.Length > maxBlockSize) { throw new ArgumentOutOfRangeException("header", "The header cannot be larger than " + maxBlockSize.ToString() + " bytes."); } while (true) { using (var ms = new MemoryStream()) { AccessCondition accessCondition; IEnumerable <ListBlockItem> blockList = Enumerable.Empty <ListBlockItem>(); try { blockList = await blob.DownloadBlockListAsync(BlockListingFilter.Committed, null, options, operationContext, cancellationToken).ConfigureAwait(false); // 404 if blob not found accessCondition = AccessCondition.GenerateIfMatchCondition(blob.Properties.ETag); // Write if blob matches what we read } catch (StorageException se) { if (!se.Matches(HttpStatusCode.NotFound, BlobErrorCodeStrings.BlobNotFound)) { throw; } accessCondition = AccessCondition.GenerateIfNoneMatchCondition("*"); // Write if blob doesn't exist } try { List <String> blockIds = blockList.Select(lbi => lbi.Name).ToList(); ListBlockItem lastBlock = blockList.LastOrDefault(); if (lastBlock == null) { if (header != null) { // No blocks exist, add header (if specified) ms.Write(header, 0, header.Length); } } else { // A block exists, can it hold the new data? if (lastBlock.Length + data.Length < maxBlockSize) { // Yes, download the last block's current data as long as the blob's etag hasn't changed Int64 lastBlockOffset = blockList.Sum(lbi => lbi.Length) - lastBlock.Length; await blob.DownloadRangeToStreamAsync(ms, lastBlockOffset, lastBlock.Length, accessCondition, options, operationContext, cancellationToken).ConfigureAwait(false); // 412 if blob modified behind our back blockIds.Remove(lastBlock.Name); // Remove the block we're appending to } } await data.CopyToAsync(ms).ConfigureAwait(false); // Append new data to end of stream ms.Seek(0, SeekOrigin.Begin); // Upload new block and append it to the blob String newBlockId = Guid.NewGuid().ToString("N").Encode().ToBase64String(); blockIds.Add(newBlockId); // Append new block to end of blob await blob.PutBlockAsync(newBlockId, ms, null, accessCondition, options, operationContext, cancellationToken).ConfigureAwait(false); // PutBlock ignores access condition so this always succeeds await blob.PutBlockListAsync(blockIds, accessCondition, options, operationContext, cancellationToken).ConfigureAwait(false); // 409 if blob created behind our back; 400 if block ID doesn't exist (happens in another PC calls PutBlockList after our PutBlock) break; // If successful, we're done; don't retry } catch (StorageException se) { // Blob got created behind our back, retry if (se.Matches(HttpStatusCode.Conflict, BlobErrorCodeStrings.BlobAlreadyExists)) { continue; } // Blob got created or modified behind our back, retry if (se.Matches(HttpStatusCode.PreconditionFailed)) { continue; } // Another PC called PutBlockList between our PutBlock & PutBlockList, // our block got destroyed, retry if (se.Matches(HttpStatusCode.BadRequest, BlobErrorCodeStrings.InvalidBlockList)) { continue; } throw; } } } }