public static async Task DownloadBlobData(CloudBlockBlob blob)
{
var info = blob.Properties;
var packets = (info.Length - 6) / 128;
var bytes = new byte[128];
var data = new List <byte[]>();
for (int packet = 0; packet < packets; packet++)
{
try
{
await blob.DownloadRangeToByteArrayAsync(bytes, 0, packet * 128, 128);
data.Add(bytes);
bytes = new byte[128];
}
catch (Exception ex)
{
Debug.WriteLine("Download failed: " + ex.Message);
return;
}
}
App.DebugMsg($"Downloaded data successfully for {blob.Name}");
// Mask first byte to indicate cloud storage.
data[0][0] |= 0x1F;
FileManager.WriteFile(data, blob.Name);
}
public override async Task <int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
//see https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-the-range-header-for-blob-service-operations#range-header-formats
int read = await _blob.DownloadRangeToByteArrayAsync(buffer, offset, Position, count);
Position += read;
return(read);
}
private async Task Tail(string fileName)
{
CloudBlockBlob cbb = _provider.NativeBlobContainer.GetBlockBlobReference(fileName);
await cbb.FetchAttributesAsync();
long length = cbb.Properties.Length - _offset;
if (length > 0)
{
byte[] buffer = new byte[length];
await cbb.DownloadRangeToByteArrayAsync(buffer, 0, _offset, buffer.Length);
_offset = cbb.Properties.Length;
List <AzureBlobEvent> events = ToBlobEvents(buffer);
foreach (AzureBlobEvent e in events)
{
Console.WriteLine(e);
}
}
}
public static async Task Run(
[BlobTrigger("%blobContainerName%/resourceId=/SUBSCRIPTIONS/{subId}/RESOURCEGROUPS/{resourceGroup}/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/{nsgName}/y={blobYear}/m={blobMonth}/d={blobDay}/h={blobHour}/m={blobMinute}/macAddress={mac}/PT1H.json", Connection = "nsgSourceDataConnection")] CloudBlockBlob myBlob,
[Table("checkpoints", Connection = "AzureWebJobsStorage")] CloudTable checkpointTable,
Binder nsgDataBlobBinder,
Binder cefLogBinder,
string subId, string resourceGroup, string nsgName, string blobYear, string blobMonth, string blobDay, string blobHour, string blobMinute, string mac,
ExecutionContext executionContext,
ILogger log)
{
log.LogDebug($"BlobTriggerIngestAndTransmit triggered: {executionContext.InvocationId} ");
string nsgSourceDataAccount = Util.GetEnvironmentVariable("nsgSourceDataAccount");
if (nsgSourceDataAccount.Length == 0)
{
log.LogError("Value for nsgSourceDataAccount is required.");
throw new System.ArgumentNullException("nsgSourceDataAccount", "Please provide setting.");
}
string blobContainerName = Util.GetEnvironmentVariable("blobContainerName");
if (blobContainerName.Length == 0)
{
log.LogError("Value for blobContainerName is required.");
throw new System.ArgumentNullException("blobContainerName", "Please provide setting.");
}
string outputBinding = Util.GetEnvironmentVariable("outputBinding");
if (outputBinding.Length == 0)
{
log.LogError("Value for outputBinding is required. Permitted values are: 'arcsight', 'splunk', 'eventhub'.");
throw new System.ArgumentNullException("outputBinding", "Please provide setting.");
}
var blobDetails = new BlobDetails(subId, resourceGroup, nsgName, blobYear, blobMonth, blobDay, blobHour, blobMinute, mac);
// get checkpoint
Checkpoint checkpoint = Checkpoint.GetCheckpoint(blobDetails, checkpointTable);
var blockList = myBlob.DownloadBlockListAsync().Result;
var startingByte = blockList.Where((item, index) => index < checkpoint.CheckpointIndex).Sum(item => item.Length);
var endingByte = blockList.Where((item, index) => index < blockList.Count() - 1).Sum(item => item.Length);
var dataLength = endingByte - startingByte;
log.LogDebug("Blob: {0}, starting byte: {1}, ending byte: {2}, number of bytes: {3}", blobDetails.ToString(), startingByte, endingByte, dataLength);
if (dataLength == 0)
{
log.LogWarning(string.Format("Blob: {0}, triggered on completed hour.", blobDetails.ToString()));
return;
}
//foreach (var item in blockList)
//{
// log.LogInformation("Name: {0}, Length: {1}", item.Name, item.Length);
//}
var attributes = new Attribute[]
{
new BlobAttribute(string.Format("{0}/{1}", blobContainerName, myBlob.Name)),
new StorageAccountAttribute(nsgSourceDataAccount)
};
string nsgMessagesString = "";
var bytePool = ArrayPool <byte> .Shared;
byte[] nsgMessages = bytePool.Rent((int)dataLength);
try
{
CloudBlockBlob blob = nsgDataBlobBinder.BindAsync <CloudBlockBlob>(attributes).Result;
await blob.DownloadRangeToByteArrayAsync(nsgMessages, 0, startingByte, dataLength);
if (nsgMessages[0] == ',')
{
nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages, 1, (int)(dataLength - 1));
}
else
{
nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages, 0, (int)dataLength);
}
}
catch (Exception ex)
{
log.LogError(string.Format("Error binding blob input: {0}", ex.Message));
throw ex;
}
finally
{
bytePool.Return(nsgMessages);
}
//log.LogDebug(nsgMessagesString);
try
{
int bytesSent = await Util.SendMessagesDownstreamAsync(nsgMessagesString, executionContext, cefLogBinder, log);
log.LogInformation($"Sending {nsgMessagesString.Length} bytes (denormalized to {bytesSent} bytes) downstream via output binding {outputBinding}.");
}
catch (Exception ex)
{
log.LogError(string.Format("SendMessagesDownstreamAsync: Error {0}", ex.Message));
throw ex;
}
checkpoint.PutCheckpoint(checkpointTable, blockList.Count() - 1);
}
public static async Task Run(
[QueueTrigger("stage2", Connection = "AzureWebJobsStorage")] Chunk inputChunk,
Binder binder,
Binder cefLogBinder,
Binder errorRecordBinder,
TraceWriter log)
{
// log.Info($"C# Queue trigger function processed: {inputChunk}");
string nsgSourceDataAccount = Util.GetEnvironmentVariable("nsgSourceDataAccount");
if (nsgSourceDataAccount.Length == 0)
{
log.Error("Value for nsgSourceDataAccount is required.");
throw new ArgumentNullException("nsgSourceDataAccount", "Please supply in this setting the name of the connection string from which NSG logs should be read.");
}
var attributes = new Attribute[]
{
new BlobAttribute(inputChunk.BlobName),
new StorageAccountAttribute(nsgSourceDataAccount)
};
string nsgMessagesString;
try
{
byte[] nsgMessages = new byte[inputChunk.Length];
CloudBlockBlob blob = await binder.BindAsync <CloudBlockBlob>(attributes);
await blob.DownloadRangeToByteArrayAsync(nsgMessages, 0, inputChunk.Start, inputChunk.Length);
nsgMessagesString = System.Text.Encoding.UTF8.GetString(nsgMessages);
}
catch (Exception ex)
{
log.Error(string.Format("Error binding blob input: {0}", ex.Message));
throw ex;
}
// skip past the leading comma
string trimmedMessages = nsgMessagesString.Trim();
int curlyBrace = trimmedMessages.IndexOf('{');
string newClientContent = "{\"records\":[";
newClientContent += trimmedMessages.Substring(curlyBrace);
newClientContent += "]}";
await SendMessagesDownstream(newClientContent, log);
string logOutgoingCEF = Util.GetEnvironmentVariable("logOutgoingCEF");
Boolean flag;
if (Boolean.TryParse(logOutgoingCEF, out flag))
{
if (flag)
{
await CEFLog(newClientContent, cefLogBinder, errorRecordBinder, log);
}
}
}
static void Main(string[] args)
{
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json");
Configuration = builder.Build();
CloudStorageAccount account = CloudStorageAccount.Parse(Configuration["AppSettings:storageConnection"]);
CloudQueueClient queueClient = account.CreateCloudQueueClient();
CloudQueue queue = queueClient.GetQueueReference("blobcreateevents");
queue.CreateIfNotExistsAsync();
CloudBlobClient blobClient = account.CreateCloudBlobClient();
bool done = false;
ListBlockItem lastBlockDownloaded = null;
int currentBackOff = 0;
int maxBackOff = 5;
byte[] bytesFromBlob = new byte[4 * 1024 * 1024];
do
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Getting messages from the queue.");
queue.FetchAttributesAsync().Wait();
int countOfMessages = Convert.ToInt32(queue.ApproximateMessageCount);
if (countOfMessages > 0)
{
currentBackOff = 1;
var msgs = queue.GetMessagesAsync(countOfMessages > 32 ? 32 : countOfMessages).Result;
var numberOfMessagesLeftInBatch = msgs.Count();
foreach (var msg in msgs)
{
if (numberOfMessagesLeftInBatch == 1)
// process only the last message
// all messages are equivalent in meaning
// sender usually gets ahead of receiver
{
JToken token = JObject.Parse(msg.AsString);
string resourceId = (string)token.SelectToken("topic");
string subject = (string)token.SelectToken("subject");
string eventType = (string)token.SelectToken("eventType");
if (eventType == "Microsoft.Storage.BlobCreated")
{
var storageAccountName = resourceId.Split('/')[8];
var storageContainerName = subject.Split('/')[4];
var blobName = subject.Split('/')[6];
Uri uri = new Uri($"https://{storageAccountName}.blob.core.windows.net/{storageContainerName}/{blobName}");
CloudBlockBlob blob = new CloudBlockBlob(uri, blobClient);
var blockList = blob.DownloadBlockListAsync().Result;
long blobOffset = 0;
var blocksToCopy = blockList;
if (lastBlockDownloaded != null)
{
// count the blocks already written
var countOfblocksAlreadyWritten = blockList.TakeWhile(item => (item.Name != lastBlockDownloaded.Name)).Count() + 1;
// get an enumerable of those block list items
var blocksAlreadyWritten = blockList.Take(countOfblocksAlreadyWritten);
// add up the bytes already written
foreach (var block in blocksAlreadyWritten)
{
blobOffset += block.Length;
}
// skip over blocks already written
blocksToCopy = blockList.SkipWhile(item => (item.Name != lastBlockDownloaded.Name)).Skip(1);
}
if (blocksToCopy.Count() > 0)
{
var fs = File.OpenWrite(@"c:\temp\abigfile.dat");
using (BinaryWriter writer = new BinaryWriter(fs))
{
foreach (var block in blocksToCopy)
{
blob.DownloadRangeToByteArrayAsync(bytesFromBlob, 0, blobOffset, block.Length).Wait();
writer.Seek(0, SeekOrigin.End);
writer.Write(bytesFromBlob, 0, (int)block.Length);
blobOffset += block.Length;
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine($"{block.Name}");
}
};
fs.Close();
}
lastBlockDownloaded = blockList.Last();
}
}
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Deleting a message from the queue");
queue.DeleteMessageAsync(msg);
numberOfMessagesLeftInBatch--;
}
}
else
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"Waiting for {currentBackOff} seconds for next message");
Thread.Sleep(currentBackOff * 1000);
currentBackOff += (currentBackOff < maxBackOff ? 1 : 0);
}
// set done flag in some way, or loop forever.
} while (!done);
Console.ReadKey();
}
public async Task <IActionResult> Zip(string ids, bool?compressed, CancellationToken token)
{
//var r = BadRequestIfPasswordInvalid();
//if (r != null) return r;
Response.ContentType = "application/zip";
Response.Headers.Add("Content-Disposition", $"attachment;filename=photobin_{DateTime.UtcNow.ToString("yyyyMMdd-hhmmss")}.zip");
IEnumerable <Guid> guids = ids.Split('\r', '\n', ',', ';').Where(s => s != "").Select(s => Guid.Parse(s));
var fileMetadata = await _context.FileMetadata.Where(f => guids.Contains(f.FileMetadataId)).ToListAsync();
var compressionLevel = compressed == true
? CompressionLevel.Optimal
: CompressionLevel.NoCompression;
if (compressionLevel == CompressionLevel.NoCompression)
{
// The size of a .zip file with no compression can be determined from just the names and sizes of the files.
using (var s = new LengthStream()) {
using (var archive = new ZipArchive(s, ZipArchiveMode.Create, true)) {
foreach (var file in fileMetadata)
{
token.ThrowIfCancellationRequested();
byte[] data = new byte[file.Size];
var entry = archive.CreateEntry(file.NewFilename, compressionLevel);
entry.LastWriteTime = file.UploadedAt;
using (var zipStream = entry.Open()) {
await zipStream.WriteAsync(data, 0, data.Length);
}
}
}
Response.Headers.Add("Content-Length", s.Position.ToString());
}
}
using (var s = Response.Body) {
using (var archive = new ZipArchive(s, ZipArchiveMode.Create, true)) {
foreach (var file in fileMetadata)
{
token.ThrowIfCancellationRequested();
var container = await GetCloudBlobContainerAsync();
CloudBlockBlob full = container.GetBlockBlobReference($"full-{file.FileMetadataId}");
if (!await full.ExistsAsync())
{
throw new Exception($"{file.FileMetadataId} not found");
}
var entry = archive.CreateEntry(file.NewFilename, compressionLevel);
entry.LastWriteTime = file.UploadedAt;
byte[] data = new byte[1024 * 1024 * 8];
using (var zipStream = entry.Open())
{
for (int offset = 0; offset < file.Size; offset += data.Length)
{
int length = Math.Min(data.Length, file.Size - offset);
await full.DownloadRangeToByteArrayAsync(data, 0, offset, length);
await zipStream.WriteAsync(data, 0, length);
}
}
}
}
}
return(new EmptyResult());
}
public static async Task Run(
[QueueTrigger("stage1", Connection = "AzureWebJobsStorage")] Chunk inputChunk,
[Queue("stage2", Connection = "AzureWebJobsStorage")] ICollector <Chunk> outputQueue,
Binder binder,
TraceWriter log)
{
// log.Info($"C# Queue trigger function processed: {inputChunk}");
if (inputChunk.Length < MAX_CHUNK_SIZE)
{
outputQueue.Add(inputChunk);
return;
}
string nsgSourceDataAccount = Util.GetEnvironmentVariable("nsgSourceDataAccount");
if (nsgSourceDataAccount.Length == 0)
{
log.Error("Value for nsgSourceDataAccount is required.");
throw new ArgumentNullException("nsgSourceDataAccount", "Please supply in this setting the name of the connection string from which NSG logs should be read.");
}
var attributes = new Attribute[]
{
new BlobAttribute(inputChunk.BlobName),
new StorageAccountAttribute(nsgSourceDataAccount)
};
byte[] nsgMessages = new byte[inputChunk.Length];
try
{
CloudBlockBlob blob = await binder.BindAsync <CloudBlockBlob>(attributes);
await blob.DownloadRangeToByteArrayAsync(nsgMessages, 0, inputChunk.Start, inputChunk.Length);
}
catch (Exception ex)
{
log.Error(string.Format("Error binding blob input: {0}", ex.Message));
throw ex;
}
int startingByte = 0;
var chunkCount = 0;
var newChunk = GetNewChunk(inputChunk, chunkCount++, log, 0);
//long length = FindNextRecord(nsgMessages, startingByte);
var nsgMessagesString = System.Text.Encoding.Default.GetString(nsgMessages);
int endingByte = FindNextRecordRecurse(nsgMessagesString, startingByte, 0, log);
int length = endingByte - startingByte + 1;
while (length != 0)
{
if (newChunk.Length + length > MAX_CHUNK_SIZE)
{
//log.Info($"Chunk starts at {newChunk.Start}, length is {newChunk.Length}, next start is {newChunk.Start + newChunk.Length}");
outputQueue.Add(newChunk);
newChunk = GetNewChunk(inputChunk, chunkCount++, log, newChunk.Start + newChunk.Length);
}
newChunk.Length += length;
startingByte += length;
endingByte = FindNextRecordRecurse(nsgMessagesString, startingByte, 0, log);
length = endingByte - startingByte + 1;
}
if (newChunk.Length > 0)
{
outputQueue.Add(newChunk);
//log.Info($"Chunk starts at {newChunk.Start}, length is {newChunk.Length}");
}
}
public int ReadContentInto(long pos, byte[] buffer, int offset, int size)
{
return(_blob.DownloadRangeToByteArrayAsync(buffer, offset, pos, size).Result);
}
