/// <summary> /// input newClientContent is a string representation of a json array of records, each of which is a nsg flow log hierarchy /// output is a List of SplunkEventMessage, up to a max # of bytes or 450 elements /// </summary> /// <param name="newClientContent"></param> /// <param name="errorRecordBinder"></param> /// <param name="log"></param> /// <returns></returns> static IEnumerable <List <SplunkEventMessage> > denormalizedSplunkEvents(string newClientContent, Binder errorRecordBinder, ILogger log) { var outgoingSplunkList = ListPool <SplunkEventMessage> .Allocate(); outgoingSplunkList.Capacity = 450; var sizeOfListItems = 0; try { NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent); foreach (var record in logs.records) { float version = record.properties.Version; foreach (var outerFlow in record.properties.flows) { foreach (var innerFlow in outerFlow.flows) { foreach (var flowTuple in innerFlow.flowTuples) { var tuple = new NSGFlowLogTuple(flowTuple, version); var denormalizedRecord = new DenormalizedRecord( record.properties.Version, record.time, record.category, record.operationName, record.resourceId, outerFlow.rule, innerFlow.mac, tuple); var splunkEventMessage = new SplunkEventMessage(denormalizedRecord); var sizeOfObject = splunkEventMessage.GetSizeOfObject(); if (sizeOfListItems + sizeOfObject > MAXTRANSMISSIONSIZE + 20 || outgoingSplunkList.Count == 450) { yield return(outgoingSplunkList); outgoingSplunkList.Clear(); sizeOfListItems = 0; } outgoingSplunkList.Add(splunkEventMessage); sizeOfListItems += sizeOfObject; } } } } if (sizeOfListItems > 0) { yield return(outgoingSplunkList); } } finally { ListPool <SplunkEventMessage> .Free(outgoingSplunkList); } }
public Source(DenormalizedRecord denormalizedRecord) { this.address = denormalizedRecord.sourceAddress; this.ip = denormalizedRecord.sourceAddress; this.port = Convert.ToInt64(denormalizedRecord.sourcePort); this.packets = Convert.ToInt64(denormalizedRecord.packetsStoD); this.bytes = Convert.ToInt64(denormalizedRecord.bytesStoD); }
public Destination(DenormalizedRecord denormalizedRecord) { this.address = denormalizedRecord.destinationAddress; this.ip = denormalizedRecord.destinationAddress; this.port = Convert.ToInt64(denormalizedRecord.destinationPort); this.packets = Convert.ToInt64(denormalizedRecord.packetsDtoS); this.bytes = Convert.ToInt64(denormalizedRecord.bytesDtoS); }
public Network(DenormalizedRecord denormalizedRecord) { this.transport = (denormalizedRecord.transportProtocol == "U" ? "udp" : "tcp"); this.direction = (denormalizedRecord.deviceDirection == "I" ? "inbound" : "outbound"); this.protocol = "transport"; this.bytes = Convert.ToInt64(denormalizedRecord.bytesStoD) + Convert.ToInt64(denormalizedRecord.bytesDtoS); this.packets = Convert.ToInt64(denormalizedRecord.packetsStoD) + Convert.ToInt64(denormalizedRecord.packetsDtoS); this.flowstate = denormalizedRecord.flowState; }
public void denormalizedRecordJsonTest() { DenormalizedRecord denormalizedRecordV2 = EcsTest.createDenormalizedRecordV2(); var outgoingJson = JsonConvert.SerializeObject(denormalizedRecordV2, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); String expected = "{\"time\":\"2020-01-15T07:00:00.5173253Z\",\"category\":\"NetworkSecurityGroupFlowEvent\",\"operationName\":\"NetworkSecurityGroupFlowEvents\",\"resourceId\":\"/SUBSCRIPTIONS/F087A016-314D-482C-93F1-88665DAFBA23/RESOURCEGROUPS/MC_MDRNWRK-DEV-AKS-RESOURCES_MDRNWRK-DEV-AKS_UKSOUTH/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/AKS-AGENTPOOL-14244569-NSG\",\"version\":2.0,\"nsgRuleName\":\"DefaultRule_AllowVnetOutBound\",\"mac\":\"000D3R5F1340\",\"startTime\":\"1578673962\",\"sourceAddress\":\"10.244.0.40\",\"destinationAddress\":\"10.244.1.68\",\"sourcePort\":\"36098\",\"destinationPort\":\"25227\",\"transportProtocol\":\"T\",\"deviceDirection\":\"I\",\"deviceAction\":\"A\",\"flowState\":\"E\",\"packetsStoD\":\"3\",\"bytesStoD\":\"206\",\"packetsDtoS\":\"2\",\"bytesDtoS\":\"140\"}"; Assert.Equal(expected, outgoingJson); }
public EcsAll(DenormalizedRecord denormalizedRecord) { this.timestamp = denormalizedRecord.time; this.agent = new Agent("AzureNetworkWatcherNSGFlowLogsConnector"); this.rule = new Rule(denormalizedRecord.nsgRuleName); this.ecs = new Ecs("1.0.0"); this.type = "ecs-flowlogs-azurensg"; this.client = new Client(denormalizedRecord.mac); this.ecsevent = new EcsEvent(denormalizedRecord); this.resource = new Resource(denormalizedRecord.resourceId); this.source = new Source(denormalizedRecord); this.destination = new Destination(denormalizedRecord); this.network = new Network(denormalizedRecord); }
private static DenormalizedRecord createDenormalizedRecord2V2() { NSGFlowLogTuple tuple = new NSGFlowLogTuple("1578673962,10.244.0.40,10.244.1.68,36098,25227,U,O,D,C,10,1300,9,600", 2.0f); DenormalizedRecord record = new DenormalizedRecord(2.0f, "2020-01-15T07:00:00.5173253Z", "NetworkSecurityGroupFlowEvent", "NetworkSecurityGroupFlowEvents", "/SUBSCRIPTIONS/F087A016-314D-482C-93F1-88665DAFBA23/RESOURCEGROUPS/MC_MDRNWRK-DEV-AKS-RESOURCES_MDRNWRK-DEV-AKS_UKSOUTH/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/AKS-AGENTPOOL-14244569-NSG", "DefaultRule_AllowVnetOutBound", "000D3R5F1340", tuple); return(record); }
public EcsEvent(DenormalizedRecord denormalizedRecord) { this.category = denormalizedRecord.category; this.action = denormalizedRecord.operationName; this.outcome = (denormalizedRecord.deviceAction == "A" ? "allowed" : "denied"); DateTimeOffset dateTimeOffset = DateTimeOffset.FromUnixTimeSeconds(Convert.ToInt64(denormalizedRecord.startTime)); this.start = dateTimeOffset.UtcDateTime.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ"); this.dataset = "nsg.access"; DateTime ingestedUtcNow = DateTime.UtcNow; this.ingested = ingestedUtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ"); }
public void denormalizedRecordToEcsTest() { DenormalizedRecord denormalizedRecordV2 = EcsTest.createDenormalizedRecordV2(); EcsAll ecsAll = EcsFactory.createEcsAll(denormalizedRecordV2); var outgoingJson = JsonConvert.SerializeObject(ecsAll, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore, Formatting = Newtonsoft.Json.Formatting.Indented, }); output.WriteLine(outgoingJson); Assert.Equal(denormalizedRecordV2.time, ecsAll.@timestamp); Assert.Equal("AzureNetworkWatcherNSGFlowLogsConnector", ecsAll.agent.name); Assert.Equal(denormalizedRecordV2.nsgRuleName, ecsAll.rule.name); Assert.Equal("1.0.0", ecsAll.ecs.version); Assert.Equal(denormalizedRecordV2.mac, ecsAll.client.mac); Assert.Equal(denormalizedRecordV2.category, ecsAll.ecsevent.category); Assert.Equal(denormalizedRecordV2.operationName, ecsAll.ecsevent.action); Assert.Equal("allowed", ecsAll.ecsevent.outcome); Assert.Equal("nsg.access", ecsAll.ecsevent.dataset); Assert.Equal("2020-01-10T16:32:42.0000000Z", ecsAll.ecsevent.start); Assert.Equal(denormalizedRecordV2.resourceId, ecsAll.resource.id); Assert.Equal("F087A016-314D-482C-93F1-88665DAFBA23", ecsAll.resource.subscription); Assert.Equal("AKS-AGENTPOOL-14244569-NSG", ecsAll.resource.nsg); Assert.Equal("10.244.0.40", ecsAll.source.address); Assert.Equal("10.244.0.40", ecsAll.source.ip); Assert.Equal("10.244.1.68", ecsAll.destination.address); Assert.Equal("10.244.1.68", ecsAll.destination.ip); Assert.Equal(36098, ecsAll.source.port); Assert.Equal(25227, ecsAll.destination.port); Assert.Equal(3, ecsAll.source.packets); Assert.Equal(2, ecsAll.destination.packets); Assert.Equal(206, ecsAll.source.bytes); Assert.Equal(140, ecsAll.destination.bytes); Assert.Equal("tcp", ecsAll.network.transport); Assert.Equal("inbound", ecsAll.network.direction); Assert.Equal("transport", ecsAll.network.protocol); Assert.Equal(5, ecsAll.network.packets); Assert.Equal(346, ecsAll.network.bytes); Assert.Equal("E", ecsAll.network.flowstate); }
public SplunkEventMessage(DenormalizedRecord splunkEvent) { sourcetype = "amdl:nsg:flowlogs"; time = unixTime(splunkEvent.time); @event = splunkEvent; }
static IEnumerable <List <DenormalizedRecord> > denormalizedRecords(string newClientContent, Binder errorRecordBinder, ILogger log) { var outgoingList = ListPool <DenormalizedRecord> .Allocate(); outgoingList.Capacity = 450; var sizeOfListItems = 0; try { NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent); foreach (var record in logs.records) { float version = record.properties.Version; foreach (var outerFlow in record.properties.flows) { foreach (var innerFlow in outerFlow.flows) { foreach (var flowTuple in innerFlow.flowTuples) { var tuple = new NSGFlowLogTuple(flowTuple, version); var denormalizedRecord = new DenormalizedRecord( record.properties.Version, record.time, record.category, record.operationName, record.resourceId, outerFlow.rule, innerFlow.mac, tuple); var sizeOfDenormalizedRecord = denormalizedRecord.GetSizeOfJSONObject(); //for Event hub binding fork -- start // Event hub basic message size is 256KB and the 'if' statement below ensures that list does not exceed size this size for Eventhub string outputBinding = Util.GetEnvironmentVariable("outputBinding"); if (outputBinding == "eventhub") { if (sizeOfListItems > 120) // this will chunk below 256KB : this is ideal sample message size. Feel free to go maximum till 150 : smaller values will create lot of outbound connections. { yield return(outgoingList); outgoingList.Clear(); sizeOfListItems = 0; } outgoingList.Add(denormalizedRecord); sizeOfListItems += 1; } //for Event hub binding fork -- end //other output bindings else if (sizeOfListItems + sizeOfDenormalizedRecord > MAXTRANSMISSIONSIZE + 20) { yield return(outgoingList); outgoingList.Clear(); sizeOfListItems = 0; } outgoingList.Add(denormalizedRecord); sizeOfListItems += sizeOfDenormalizedRecord; } } } } if (sizeOfListItems > 0) { yield return(outgoingList); } } finally { ListPool <DenormalizedRecord> .Free(outgoingList); } }
public SplunkEventMessage(DenormalizedRecord splunkEvent) { sourcetype = "amdl:nsg:flowlogs"; @event = splunkEvent; }
public static EcsAll createEcsAll(DenormalizedRecord denormalizedRecord) { EcsAll ecsAll = new EcsAll(denormalizedRecord); return(ecsAll); }
static IEnumerable <List <DenormalizedRecord> > denormalizedRecords(string newClientContent, Binder errorRecordBinder, ILogger log) { var outgoingList = ListPool <DenormalizedRecord> .Allocate(); outgoingList.Capacity = 450; var sizeOfListItems = 0; try { NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent); foreach (var record in logs.records) { float version = record.properties.Version; foreach (var outerFlow in record.properties.flows) { foreach (var innerFlow in outerFlow.flows) { foreach (var flowTuple in innerFlow.flowTuples) { var tuple = new NSGFlowLogTuple(flowTuple, version); var denormalizedRecord = new DenormalizedRecord( record.properties.Version, record.time, record.category, record.operationName, record.resourceId, outerFlow.rule, innerFlow.mac, tuple); var sizeOfDenormalizedRecord = denormalizedRecord.GetSizeOfJSONObject(); //new-code-start //if (sizeOfDenormalizedRecord > 120) // this will chunk below 256KB if (sizeOfListItems > 120) { yield return(outgoingList); outgoingList.Clear(); sizeOfListItems = 0; } //new-code-end /* * if (sizeOfListItems + sizeOfDenormalizedRecord > MAXTRANSMISSIONSIZE + 20) * { * yield return outgoingList; * outgoingList.Clear(); * sizeOfListItems = 0; * } */ outgoingList.Add(denormalizedRecord); //sizeOfListItems += sizeOfDenormalizedRecord; sizeOfListItems += 1; } } } } if (sizeOfListItems > 0) { yield return(outgoingList); } } finally { ListPool <DenormalizedRecord> .Free(outgoingList); } }