public static async Task DownloadBlobData(CloudBlockBlob blob) { var info = blob.Properties; var packets = (info.Length - 6) / 128; var bytes = new byte[128]; var data = new List <byte[]>(); for (int packet = 0; packet < packets; packet++) { try { await blob.DownloadRangeToByteArrayAsync(bytes, 0, packet * 128, 128); data.Add(bytes); bytes = new byte[128]; } catch (Exception ex) { Debug.WriteLine("Download failed: " + ex.Message); return; } } App.DebugMsg($"Downloaded data successfully for {blob.Name}"); // Mask first byte to indicate cloud storage. data[0][0] |= 0x1F; FileManager.WriteFile(data, blob.Name); }
public override async Task <int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { //see https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-the-range-header-for-blob-service-operations#range-header-formats int read = await _blob.DownloadRangeToByteArrayAsync(buffer, offset, Position, count); Position += read; return(read); }
private async Task Tail(string fileName) { CloudBlockBlob cbb = _provider.NativeBlobContainer.GetBlockBlobReference(fileName); await cbb.FetchAttributesAsync(); long length = cbb.Properties.Length - _offset; if (length > 0) { byte[] buffer = new byte[length]; await cbb.DownloadRangeToByteArrayAsync(buffer, 0, _offset, buffer.Length); _offset = cbb.Properties.Length; List <AzureBlobEvent> events = ToBlobEvents(buffer); foreach (AzureBlobEvent e in events) { Console.WriteLine(e); } } }
public static async Task Run( [BlobTrigger("%blobContainerName%/resourceId=/SUBSCRIPTIONS/{subId}/RESOURCEGROUPS/{resourceGroup}/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/{nsgName}/y={blobYear}/m={blobMonth}/d={blobDay}/h={blobHour}/m={blobMinute}/macAddress={mac}/PT1H.json", Connection = "nsgSourceDataConnection")] CloudBlockBlob myBlob, [Table("checkpoints", Connection = "AzureWebJobsStorage")] CloudTable checkpointTable, Binder nsgDataBlobBinder, Binder cefLogBinder, string subId, string resourceGroup, string nsgName, string blobYear, string blobMonth, string blobDay, string blobHour, string blobMinute, string mac, ExecutionContext executionContext, ILogger log) { log.LogDebug($"BlobTriggerIngestAndTransmit triggered: {executionContext.InvocationId} "); string nsgSourceDataAccount = Util.GetEnvironmentVariable("nsgSourceDataAccount"); if (nsgSourceDataAccount.Length == 0) { log.LogError("Value for nsgSourceDataAccount is required."); throw new System.ArgumentNullException("nsgSourceDataAccount", "Please provide setting."); } string blobContainerName = Util.GetEnvironmentVariable("blobContainerName"); if (blobContainerName.Length == 0) { log.LogError("Value for blobContainerName is required."); throw new System.ArgumentNullException("blobContainerName", "Please provide setting."); } string outputBinding = Util.GetEnvironmentVariable("outputBinding"); if (outputBinding.Length == 0) { log.LogError("Value for outputBinding is required. Permitted values are: 'arcsight', 'splunk', 'eventhub'."); throw new System.ArgumentNullException("outputBinding", "Please provide setting."); } var blobDetails = new BlobDetails(subId, resourceGroup, nsgName, blobYear, blobMonth, blobDay, blobHour, blobMinute, mac); // get checkpoint Checkpoint checkpoint = Checkpoint.GetCheckpoint(blobDetails, checkpointTable); var blockList = myBlob.DownloadBlockListAsync().Result; var startingByte = blockList.Where((item, index) => index < checkpoint.CheckpointIndex).Sum(item => item.Length); var endingByte = blockList.Where((item, index) => index < blockList.Count() - 1).Sum(item => item.Length); var dataLength = endingByte - startingByte; log.LogDebug("Blob: {0}, starting byte: {1}, ending byte: {2}, number of bytes: {3}", blobDetails.ToString(), startingByte, endingByte, dataLength); if (dataLength == 0) { log.LogWarning(string.Format("Blob: {0}, triggered on completed hour.", blobDetails.ToString())); return; } //foreach (var item in blockList) //{ // log.LogInformation("Name: {0}, Length: {1}", item.Name, item.Length); //} var attributes = new Attribute[] { new BlobAttribute(string.Format("{0}/{1}", blobContainerName, myBlob.Name)), new StorageAccountAttribute(nsgSourceDataAccount) }; string nsgMessagesString = ""; var bytePool = ArrayPool <byte> .Shared; byte[] nsgMessages = bytePool.Rent((int)dataLength); try { CloudBlockBlob blob = nsgDataBlobBinder.BindAsync <CloudBlockBlob>(attributes).Result; await blob.DownloadRangeToByteArrayAsync(nsgMessages, 0, startingByte, dataLength); if (nsgMessages[0] == ',') { nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages, 1, (int)(dataLength - 1)); } else { nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages, 0, (int)dataLength); } } catch (Exception ex) { log.LogError(string.Format("Error binding blob input: {0}", ex.Message)); throw ex; } finally { bytePool.Return(nsgMessages); } //log.LogDebug(nsgMessagesString); try { int bytesSent = await Util.SendMessagesDownstreamAsync(nsgMessagesString, executionContext, cefLogBinder, log); log.LogInformation($"Sending {nsgMessagesString.Length} bytes (denormalized to {bytesSent} bytes) downstream via output binding {outputBinding}."); } catch (Exception ex) { log.LogError(string.Format("SendMessagesDownstreamAsync: Error {0}", ex.Message)); throw ex; } checkpoint.PutCheckpoint(checkpointTable, blockList.Count() - 1); }
public static async Task Run( [QueueTrigger("stage2", Connection = "AzureWebJobsStorage")] Chunk inputChunk, Binder binder, Binder cefLogBinder, Binder errorRecordBinder, TraceWriter log) { // log.Info($"C# Queue trigger function processed: {inputChunk}"); string nsgSourceDataAccount = Util.GetEnvironmentVariable("nsgSourceDataAccount"); if (nsgSourceDataAccount.Length == 0) { log.Error("Value for nsgSourceDataAccount is required."); throw new ArgumentNullException("nsgSourceDataAccount", "Please supply in this setting the name of the connection string from which NSG logs should be read."); } var attributes = new Attribute[] { new BlobAttribute(inputChunk.BlobName), new StorageAccountAttribute(nsgSourceDataAccount) }; string nsgMessagesString; try { byte[] nsgMessages = new byte[inputChunk.Length]; CloudBlockBlob blob = await binder.BindAsync <CloudBlockBlob>(attributes); await blob.DownloadRangeToByteArrayAsync(nsgMessages, 0, inputChunk.Start, inputChunk.Length); nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages); } catch (Exception ex) { log.Error(string.Format("Error binding blob input: {0}", ex.Message)); throw ex; } // skip past the leading comma string trimmedMessages = nsgMessagesString.Trim(); int curlyBrace = trimmedMessages.IndexOf('{'); string newClientContent = "{\"records\":["; newClientContent += trimmedMessages.Substring(curlyBrace); newClientContent += "]}"; await SendMessagesDownstream(newClientContent, log); string logOutgoingCEF = Util.GetEnvironmentVariable("logOutgoingCEF"); Boolean flag; if (Boolean.TryParse(logOutgoingCEF, out flag)) { if (flag) { await CEFLog(newClientContent, cefLogBinder, errorRecordBinder, log); } } }
static void Main(string[] args) { var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json"); Configuration = builder.Build(); CloudStorageAccount account = CloudStorageAccount.Parse(Configuration["AppSettings:storageConnection"]); CloudQueueClient queueClient = account.CreateCloudQueueClient(); CloudQueue queue = queueClient.GetQueueReference("blobcreateevents"); queue.CreateIfNotExistsAsync(); CloudBlobClient blobClient = account.CreateCloudBlobClient(); bool done = false; ListBlockItem lastBlockDownloaded = null; int currentBackOff = 0; int maxBackOff = 5; byte[] bytesFromBlob = new byte[4 * 1024 * 1024]; do { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("Getting messages from the queue."); queue.FetchAttributesAsync().Wait(); int countOfMessages = Convert.ToInt32(queue.ApproximateMessageCount); if (countOfMessages > 0) { currentBackOff = 1; var msgs = queue.GetMessagesAsync(countOfMessages > 32 ? 32 : countOfMessages).Result; var numberOfMessagesLeftInBatch = msgs.Count(); foreach (var msg in msgs) { if (numberOfMessagesLeftInBatch == 1) // process only the last message // all messages are equivalent in meaning // sender usually gets ahead of receiver { JToken token = JObject.Parse(msg.AsString); string resourceId = (string)token.SelectToken("topic"); string subject = (string)token.SelectToken("subject"); string eventType = (string)token.SelectToken("eventType"); if (eventType == "Microsoft.Storage.BlobCreated") { var storageAccountName = resourceId.Split('/')[8]; var storageContainerName = subject.Split('/')[4]; var blobName = subject.Split('/')[6]; Uri uri = new Uri($"https://{storageAccountName}.blob.core.windows.net/{storageContainerName}/{blobName}"); CloudBlockBlob blob = new CloudBlockBlob(uri, blobClient); var blockList = blob.DownloadBlockListAsync().Result; long blobOffset = 0; var blocksToCopy = blockList; if (lastBlockDownloaded != null) { // count the blocks already written var countOfblocksAlreadyWritten = blockList.TakeWhile(item => (item.Name != lastBlockDownloaded.Name)).Count() + 1; // get an enumerable of those block list items var blocksAlreadyWritten = blockList.Take(countOfblocksAlreadyWritten); // add up the bytes already written foreach (var block in blocksAlreadyWritten) { blobOffset += block.Length; } // skip over blocks already written blocksToCopy = blockList.SkipWhile(item => (item.Name != lastBlockDownloaded.Name)).Skip(1); } if (blocksToCopy.Count() > 0) { var fs = File.OpenWrite(@"c:\temp\abigfile.dat"); using (BinaryWriter writer = new BinaryWriter(fs)) { foreach (var block in blocksToCopy) { blob.DownloadRangeToByteArrayAsync(bytesFromBlob, 0, blobOffset, block.Length).Wait(); writer.Seek(0, SeekOrigin.End); writer.Write(bytesFromBlob, 0, (int)block.Length); blobOffset += block.Length; Console.ForegroundColor = ConsoleColor.White; Console.WriteLine($"{block.Name}"); } }; fs.Close(); } lastBlockDownloaded = blockList.Last(); } } Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("Deleting a message from the queue"); queue.DeleteMessageAsync(msg); numberOfMessagesLeftInBatch--; } } else { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine($"Waiting for {currentBackOff} seconds for next message"); Thread.Sleep(currentBackOff * 1000); currentBackOff += (currentBackOff < maxBackOff ? 1 : 0); } // set done flag in some way, or loop forever. } while (!done); Console.ReadKey(); }
public async Task <IActionResult> Zip(string ids, bool?compressed, CancellationToken token) { //var r = BadRequestIfPasswordInvalid(); //if (r != null) return r; Response.ContentType = "application/zip"; Response.Headers.Add("Content-Disposition", $"attachment;filename=photobin_{DateTime.UtcNow.ToString("yyyyMMdd-hhmmss")}.zip"); IEnumerable <Guid> guids = ids.Split('\r', '\n', ',', ';').Where(s => s != "").Select(s => Guid.Parse(s)); var fileMetadata = await _context.FileMetadata.Where(f => guids.Contains(f.FileMetadataId)).ToListAsync(); var compressionLevel = compressed == true ? CompressionLevel.Optimal : CompressionLevel.NoCompression; if (compressionLevel == CompressionLevel.NoCompression) { // The size of a .zip file with no compression can be determined from just the names and sizes of the files. using (var s = new LengthStream()) { using (var archive = new ZipArchive(s, ZipArchiveMode.Create, true)) { foreach (var file in fileMetadata) { token.ThrowIfCancellationRequested(); byte[] data = new byte[file.Size]; var entry = archive.CreateEntry(file.NewFilename, compressionLevel); entry.LastWriteTime = file.UploadedAt; using (var zipStream = entry.Open()) { await zipStream.WriteAsync(data, 0, data.Length); } } } Response.Headers.Add("Content-Length", s.Position.ToString()); } } using (var s = Response.Body) { using (var archive = new ZipArchive(s, ZipArchiveMode.Create, true)) { foreach (var file in fileMetadata) { token.ThrowIfCancellationRequested(); var container = await GetCloudBlobContainerAsync(); CloudBlockBlob full = container.GetBlockBlobReference($"full-{file.FileMetadataId}"); if (!await full.ExistsAsync()) { throw new Exception($"{file.FileMetadataId} not found"); } var entry = archive.CreateEntry(file.NewFilename, compressionLevel); entry.LastWriteTime = file.UploadedAt; byte[] data = new byte[1024 * 1024 * 8]; using (var zipStream = entry.Open()) { for (int offset = 0; offset < file.Size; offset += data.Length) { int length = Math.Min(data.Length, file.Size - offset); await full.DownloadRangeToByteArrayAsync(data, 0, offset, length); await zipStream.WriteAsync(data, 0, length); } } } } } return(new EmptyResult()); }
public static async Task Run( [QueueTrigger("stage1", Connection = "AzureWebJobsStorage")] Chunk inputChunk, [Queue("stage2", Connection = "AzureWebJobsStorage")] ICollector <Chunk> outputQueue, Binder binder, TraceWriter log) { // log.Info($"C# Queue trigger function processed: {inputChunk}"); if (inputChunk.Length < MAX_CHUNK_SIZE) { outputQueue.Add(inputChunk); return; } string nsgSourceDataAccount = Util.GetEnvironmentVariable("nsgSourceDataAccount"); if (nsgSourceDataAccount.Length == 0) { log.Error("Value for nsgSourceDataAccount is required."); throw new ArgumentNullException("nsgSourceDataAccount", "Please supply in this setting the name of the connection string from which NSG logs should be read."); } var attributes = new Attribute[] { new BlobAttribute(inputChunk.BlobName), new StorageAccountAttribute(nsgSourceDataAccount) }; byte[] nsgMessages = new byte[inputChunk.Length]; try { CloudBlockBlob blob = await binder.BindAsync <CloudBlockBlob>(attributes); await blob.DownloadRangeToByteArrayAsync(nsgMessages, 0, inputChunk.Start, inputChunk.Length); } catch (Exception ex) { log.Error(string.Format("Error binding blob input: {0}", ex.Message)); throw ex; } int startingByte = 0; var chunkCount = 0; var newChunk = GetNewChunk(inputChunk, chunkCount++, log, 0); //long length = FindNextRecord(nsgMessages, startingByte); var nsgMessagesString = System.Text.Encoding.Default.GetString(nsgMessages); int endingByte = FindNextRecordRecurse(nsgMessagesString, startingByte, 0, log); int length = endingByte - startingByte + 1; while (length != 0) { if (newChunk.Length + length > MAX_CHUNK_SIZE) { //log.Info($"Chunk starts at {newChunk.Start}, length is {newChunk.Length}, next start is {newChunk.Start + newChunk.Length}"); outputQueue.Add(newChunk); newChunk = GetNewChunk(inputChunk, chunkCount++, log, newChunk.Start + newChunk.Length); } newChunk.Length += length; startingByte += length; endingByte = FindNextRecordRecurse(nsgMessagesString, startingByte, 0, log); length = endingByte - startingByte + 1; } if (newChunk.Length > 0) { outputQueue.Add(newChunk); //log.Info($"Chunk starts at {newChunk.Start}, length is {newChunk.Length}"); } }
public int ReadContentInto(long pos, byte[] buffer, int offset, int size) { return(_blob.DownloadRangeToByteArrayAsync(buffer, offset, pos, size).Result); }