public async Task SendingValidObject() { var anObject = new MySerializableClass() { MyIntAttribute = 4, MyStringAttribute = "hello", MyListAttribute = new List <string>() { "one", "two" } }; ICollector collector = new HTTPDataCollectorAPI.Collector("{Your_Workspace_Id}", "{Your_Workspace_Key}"); await collector.Collect("TestLogType", anObject); }
static void Main(string[] args) { if (args.Length != 4) { Console.WriteLine("USAGE"); Console.WriteLine("UploadToLogAnalytics.exe [workspaceId] [workspaceKey] [tableName] [jsonFileName]"); return; } var workspaceId = args[0]; var workspaceKey = args[1]; var tableName = args[2]; var jsonFilename = args[3]; var collector = new HTTPDataCollectorAPI.Collector(workspaceId, workspaceKey); dynamic lookups = JsonConvert.DeserializeObject(File.ReadAllText(jsonFilename)); uint count = 0; foreach (var entry in lookups) { try { var t = collector.Collect(tableName, JsonConvert.SerializeObject(entry)); t.Wait(); count++; } catch (Exception ex) { Console.WriteLine(ex.Message); } if (count % 5000 == 0) { Console.WriteLine(count); } } }
public async Task SendingValidPayload() { ICollector collector = new HTTPDataCollectorAPI.Collector("{Your_Workspace_Id}", "{Your_Workspace_Key}"); await collector.Collect("TestLogType", "{\"TestAttribute\":\"TestValue\"}"); }
public static void Run([TimerTrigger("0 * * * * *")] TimerInfo myTimer, ILogger log) { try { const string Format = @"<version-number>;<request-start-time>;<operation-type>;<request-status>;<http-status-code>;<end-to-end-latency-in-ms>;<server-latency-in-ms>;<authentication-type>;<requester-account-name>;<owner-account-name>;<service-type>;<request-url>;<requested-object-key>;<request-id-header>;<operation-count>;<requester-ip-address>;<request-version-header>;<request-header-size>;<request-packet-size>;<response-header-size>;<response-packet-size>;<request-content-length>;<request-md5>;<server-md5>;<etag-identifier>;<last-modified-time>;<conditions-used>;<user-agent-header>;<referrer-header>;<client-request-id>"; const string ConnectionStringFormat = "DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1};EndpointSuffix=core.windows.net"; const string container = "statedata"; const string Blob = "state"; const string FormatString = "yyyy-MM-ddTHH:mm:ss.fffffffZ"; const string LogAnalyticsTableName = "HoneyBucketLogs"; log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}"); var blobStorageConnectionString = Environment.GetEnvironmentVariable("BlobStorageConnectionString"); var blobStorageKeys = Environment.GetEnvironmentVariable("BlobStorageAccountKeys"); var logAnalyticsKey = Environment.GetEnvironmentVariable("LogAnalyticsKey"); var logAnalyticsWorkspace = Environment.GetEnvironmentVariable("LogAnalyticsWorkspace"); log.LogInformation("BlobStorageConnectionString " + blobStorageConnectionString); log.LogInformation("BlobStorageAccountKeys " + blobStorageKeys); log.LogInformation("LogAnalyticsKey " + logAnalyticsKey); log.LogInformation("LogAnalyticsWorkspace " + logAnalyticsWorkspace); foreach (var setting in new string[] { logAnalyticsWorkspace, logAnalyticsKey, blobStorageKeys, blobStorageConnectionString }) { if (string.IsNullOrWhiteSpace(setting) || setting.StartsWith("http") || setting.StartsWith("@Microsoft.KeyVault")) { log.LogError("Invalid setting detected " + setting); log.LogError("Please see https://docs.microsoft.com/en-us/azure/app-service/app-service-key-vault-references"); throw new InvalidOperationException("Invalid setting"); } } // according to the docs here: https://docs.microsoft.com/en-us/azure/storage/common/storage-analytics-logging // it can take up to an hour for logs to hit blob storage. As such need to keep a list of timestamps of the last // entry we've seen var lastLogEntryProcessedTime = new ConcurrentDictionary <string, DateTime>(); try { var lastReadTimeStr = GetData(blobStorageConnectionString, container, Blob); log.LogInformation("Read time as " + lastReadTimeStr); lastReadTimeStr.Split(";").ToList().ForEach(x => { var split = x.Split('='); lastLogEntryProcessedTime.TryAdd(split[0], DateTime.ParseExact(split[1], FormatString, null)); }); } catch (Exception ex) { log.LogError(ex, "Could not get or parse state, did you forget to create the state file? " + ex.GetType().Name); } var fields = Format.Split(';'); var fieldList = fields.Select(x => x.Substring(1, x.Length - 2)).ToList(); var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = ";", Escape = '"', IgnoreQuotes = false, HasHeaderRecord = false, }; // build a list of storage accounts to pull the diagnostic logs from var connections = blobStorageKeys .Split(";") .ToList() .ConvertAll(x => { var unameKeyStr = x.Split(':'); var kvp = new KeyValuePair <string, string>(unameKeyStr[0], unameKeyStr[1]); if (!lastLogEntryProcessedTime.ContainsKey(kvp.Key)) { lastLogEntryProcessedTime.TryAdd(kvp.Key, DateTime.UtcNow.AddDays(-2)); } return(kvp); }); log.LogInformation($"Getting streams for " + connections.Count + " connections"); var whiteListedIps = new ConcurrentDictionary <string, byte>(); var honeybucketrawlogs = new ConcurrentBag <LogMessage>(); var rawLogMessages = new ConcurrentBag <Tuple <string, string> >(); // Here we create two tasks // * getLogsTask - Get logs from blob storage // * processLogsTask - Convert logs to CSV entries and process var getLogsTask = Task.Run(() => { Parallel.ForEach(connections, (connection) => { // By storing some state we can massively reduce the amount of log processing we do, here // we only retrieve logs that are recent var from = lastLogEntryProcessedTime[connection.Key].AddHours(-1); var to = DateTime.UtcNow.AddHours(1); var blobs = LogDownloader.DownloadStorageLogs( string.Format(ConnectionStringFormat, connection.Key, connection.Value), "blob", from, // don't want to miss any logs to); log.LogInformation(string.Format("Getting {0} log from {1} to {2}", connection.Key, from, to)); foreach (var blob in blobs) { // There is no point downloading more logs if the output queue is still full of data to process // we need to back off here and wait for the queue to reduce. while (rawLogMessages.Count >= 1000) { var rand = new Random(); // back off a random time in ms. Task.Delay(rand.Next(0, 1000)).Wait(); } try { using (var reader = new StreamReader(blob.OpenRead())) // underlying stream is closed by StreamReader { var text = reader.ReadToEnd(); rawLogMessages.Add(new Tuple <string, string>(connection.Key, text)); } } catch (Exception ex) { log.LogError(ex, "Error in getLogsTask: " + ex.Message); } } log.LogInformation(string.Format("Finished getting {0} log from {1} to {2}", connection.Key, from, to)); }); }); var processLogsTask = Task.Run(() => { while (!getLogsTask.IsCompleted) { while (rawLogMessages.TryTake(out Tuple <string, string> value)) { using (var sr = new StringReader(value.Item2)) using (var csv = new CsvReader(sr, config)) { while (csv.Read()) { var logentry = new List <KeyValuePair <string, string> >(); for (int c = 0; c < fieldList.Count(); c++) { logentry.Add(new KeyValuePair <string, string>(fieldList[c], csv.GetField(c))); } //request-start-time var requestTime = logentry.Where(x => x.Key == "request-start-time").First().Value; var requestDateTime = DateTime.ParseExact(requestTime, FormatString, null); // first check to ensure we are not processing any old log entries // if we are we can bail out here if (requestDateTime <= lastLogEntryProcessedTime[value.Item1]) { return; } lastLogEntryProcessedTime.AddOrUpdate( value.Item1, requestDateTime, (key, oldValue) => { if (requestDateTime > oldValue) { return(requestDateTime); } else { return(requestDateTime); } } ); // now we have a full logentry to process var operationType = logentry.Where(x => x.Key == "operation-type").First().Value; switch (operationType) { case "PutBlob": var ip = logentry.Where(x => x.Key == "requester-ip-address").Select(kvp => kvp.Value.Substring(0, kvp.Value.IndexOf(':'))).First(); whiteListedIps.TryAdd(ip, 0); break; case "ListBlobs": case "ListContainers": case "GetBlob": case "GetBlobProperties": case "GetContainerProperties": case "GetContainerACL": var msg = new LogMessage() { RequestTime = requestTime, URL = logentry.Where(x => x.Key == "request-url").First().Value, OriginIP = logentry.Where(x => x.Key == "requester-ip-address").Select(kvp => kvp.Value.Substring(0, kvp.Value.IndexOf(':'))).First(), RequestType = logentry.Where(x => x.Key == "operation-type").First().Value, UserAgent = logentry.Where(x => x.Key == "user-agent-header").First().Value }; if (whiteListedIps.ContainsKey(msg.OriginIP)) { // already whitelisted break; } //authentication-type var authtype = logentry.Where(x => x.Key == "authentication-type").First().Value; // Microsoft Azure Storage Explorer actively enumerates all the files in the // bucket & generates a lot of log entries. We want to ignore this because if someone has my sub in their // list and opens up this tool then I get a load of FPs if (msg.UserAgent.Contains("Microsoft Azure Storage Explorer") && authtype == "authenticated") { whiteListedIps.TryAdd(msg.OriginIP, 0); } // ignore any queries for $log or with null user agents. This is internal else if (msg.URL.Contains("$log") || string.IsNullOrWhiteSpace(msg.UserAgent)) { whiteListedIps.TryAdd(msg.OriginIP, 0); } else { honeybucketrawlogs.Add(msg); } break; default: break; } } } } } }); // Runs the tasks until they complete and add the results to Azure Sentinel using (var exitEvent = new ManualResetEvent(false)) { var processResultsTask = Task.Run(() => { while (!processLogsTask.IsCompleted && honeybucketrawlogs.Count >= 0) { exitEvent.WaitOne(30 * 1000); List <LogMessage> messages = new List <LogMessage>(); while (honeybucketrawlogs.TryTake(out LogMessage message)) { messages.Add(message); } var listOfIps = messages.Where(x => !whiteListedIps.ContainsKey(x.OriginIP)).Distinct().ToList(); if (listOfIps.Count() != 0) { listOfIps.Select(x => x.OriginIP).Distinct().ToList().ForEach(x => log.LogInformation($"Found: " + x)); log.LogInformation($"Sending to LogAnalytics"); var collector = new HTTPDataCollectorAPI.Collector( logAnalyticsWorkspace, logAnalyticsKey); try { var task = collector.Collect(LogAnalyticsTableName, listOfIps); task.Wait(); } catch (Exception ex) { log.LogError($"Couldn't send to LA", ex); return; } } } }); getLogsTask.Wait(); processLogsTask.Wait(); exitEvent.Set(); processResultsTask.Wait(); } log.LogInformation($"Finished parsing logs"); // Create a new state entry so we can be sure we only process the minimum amount of data var lastReadStr = string.Join(";", lastLogEntryProcessedTime.ToList().ConvertAll(x => x.Key + "=" + x.Value.ToString(FormatString))); if (string.IsNullOrWhiteSpace(lastReadStr)) { log.LogError($"The state string is bad! skipping"); } else { WriteData(blobStorageConnectionString, container, Blob, lastReadStr); } } catch (Exception ex) { log.LogError(ex, "An unexpected exception occured"); } }