static System.Collections.Generic.IEnumerable <string> convertToCEF(string newClientContent, Binder errorRecordBinder, TraceWriter log) { // newClientContent is a json string with records NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent); string logIncomingJSON = Util.GetEnvironmentVariable("logIncomingJSON"); Boolean flag; if (Boolean.TryParse(logIncomingJSON, out flag)) { if (flag) { logErrorRecord(newClientContent, errorRecordBinder, log).Wait(); } } string cefRecordBase = ""; foreach (var record in logs.records) { float version = record.properties.Version; cefRecordBase = record.MakeCEFTime(); cefRecordBase += "|Microsoft.Network"; cefRecordBase += "|NETWORKSECURITYGROUPS"; cefRecordBase += "|" + version.ToString("0.0"); cefRecordBase += "|" + record.category; cefRecordBase += "|" + record.operationName; cefRecordBase += "|1"; // severity is always 1 cefRecordBase += "|deviceExternalId=" + record.MakeDeviceExternalID(); foreach (var outerFlows in record.properties.flows) { // expectation is that there is only ever 1 item in record.properties.flows string cefOuterFlowRecord = cefRecordBase; cefOuterFlowRecord += String.Format(" cs1={0}", outerFlows.rule); cefOuterFlowRecord += String.Format(" cs1Label=NSGRuleName"); foreach (var innerFlows in outerFlows.flows) { var cefInnerFlowRecord = cefOuterFlowRecord; var firstFlowTupleEncountered = true; foreach (var flowTuple in innerFlows.flowTuples) { var tuple = new NSGFlowLogTuple(flowTuple, version); if (firstFlowTupleEncountered) { cefInnerFlowRecord += (tuple.GetDirection == "I" ? " dmac=" : " smac=") + innerFlows.MakeMAC(); firstFlowTupleEncountered = false; } yield return(cefInnerFlowRecord + " " + tuple.ToString()); } } } } }
/// <summary> /// input newClientContent is a string representation of a json array of records, each of which is a nsg flow log hierarchy /// output is a List of SplunkEventMessage, up to a max # of bytes or 450 elements /// </summary> /// <param name="newClientContent"></param> /// <param name="errorRecordBinder"></param> /// <param name="log"></param> /// <returns></returns> static IEnumerable <List <SplunkEventMessage> > denormalizedSplunkEvents(string newClientContent, Binder errorRecordBinder, ILogger log) { var outgoingSplunkList = ListPool <SplunkEventMessage> .Allocate(); outgoingSplunkList.Capacity = 450; var sizeOfListItems = 0; try { NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent); foreach (var record in logs.records) { float version = record.properties.Version; foreach (var outerFlow in record.properties.flows) { foreach (var innerFlow in outerFlow.flows) { foreach (var flowTuple in innerFlow.flowTuples) { var tuple = new NSGFlowLogTuple(flowTuple, version); var denormalizedRecord = new DenormalizedRecord( record.properties.Version, record.time, record.category, record.operationName, record.resourceId, outerFlow.rule, innerFlow.mac, tuple); var splunkEventMessage = new SplunkEventMessage(denormalizedRecord); var sizeOfObject = splunkEventMessage.GetSizeOfObject(); if (sizeOfListItems + sizeOfObject > MAXTRANSMISSIONSIZE + 20 || outgoingSplunkList.Count == 450) { yield return(outgoingSplunkList); outgoingSplunkList.Clear(); sizeOfListItems = 0; } outgoingSplunkList.Add(splunkEventMessage); sizeOfListItems += sizeOfObject; } } } } if (sizeOfListItems > 0) { yield return(outgoingSplunkList); } } finally { ListPool <SplunkEventMessage> .Free(outgoingSplunkList); } }
static System.Collections.Generic.IEnumerable <string> convertToSplunk(string newClientContent, Binder errorRecordBinder, TraceWriter log) { // // newClientContent looks like this: // // { // "records":[ // {...}, // {...} // ... // ] // } // NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent); string logIncomingJSON = Util.GetEnvironmentVariable("logIncomingJSON"); Boolean flag; if (Boolean.TryParse(logIncomingJSON, out flag)) { if (flag) { logErrorRecord(newClientContent, errorRecordBinder, log).Wait(); } } var sbBase = new StringBuilder(); foreach (var record in logs.records) { float version = record.properties.Version; sbBase.Clear(); sbBase.Append("{"); sbBase.Append(eqs("time", true)).Append(eqs(record.time)); sbBase.Append(eqs(true, "category")).Append(eqs(record.category)); sbBase.Append(eqs(true, "operationName")).Append(eqs(record.operationName)); sbBase.Append(eqs(true, "version")).Append(eqs(version.ToString("0.0"))); sbBase.Append(eqs(true, "deviceExtId")).Append(eqs(record.MakeDeviceExternalID())); int count = 1; var sbOuterFlowRecord = new StringBuilder(); foreach (var outerFlows in record.properties.flows) { sbOuterFlowRecord.Clear(); sbOuterFlowRecord.Append(sbBase.ToString()); sbOuterFlowRecord.Append(eqs(true, "flowOrder")).Append(eqs(count.ToString())); sbOuterFlowRecord.Append(eqs(true, "nsgRuleName")).Append(eqs(outerFlows.rule)); var sbInnerFlowRecord = new StringBuilder(); foreach (var innerFlows in outerFlows.flows) { sbInnerFlowRecord.Clear(); sbInnerFlowRecord.Append(sbOuterFlowRecord.ToString()); var firstFlowTupleEncountered = true; foreach (var flowTuple in innerFlows.flowTuples) { var tuple = new NSGFlowLogTuple(flowTuple, version); if (firstFlowTupleEncountered) { sbInnerFlowRecord.Append((tuple.GetDirection == "I" ? eqs(true, "dmac") : eqs(true, "smac"))).Append(eqs(innerFlows.MakeMAC())); firstFlowTupleEncountered = false; } yield return(sbInnerFlowRecord.ToString() + tuple.JsonSubString() + "}"); } } } } }
static IEnumerable <List <DenormalizedRecord> > denormalizedRecords(string newClientContent, Binder errorRecordBinder, ILogger log) { var outgoingList = ListPool <DenormalizedRecord> .Allocate(); outgoingList.Capacity = 450; var sizeOfListItems = 0; try { NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent); foreach (var record in logs.records) { float version = record.properties.Version; foreach (var outerFlow in record.properties.flows) { foreach (var innerFlow in outerFlow.flows) { foreach (var flowTuple in innerFlow.flowTuples) { var tuple = new NSGFlowLogTuple(flowTuple, version); var denormalizedRecord = new DenormalizedRecord( record.properties.Version, record.time, record.category, record.operationName, record.resourceId, outerFlow.rule, innerFlow.mac, tuple); var sizeOfDenormalizedRecord = denormalizedRecord.GetSizeOfJSONObject(); //for Event hub binding fork -- start // Event hub basic message size is 256KB and the 'if' statement below ensures that list does not exceed size this size for Eventhub string outputBinding = Util.GetEnvironmentVariable("outputBinding"); if (outputBinding == "eventhub") { if (sizeOfListItems > 120) // this will chunk below 256KB : this is ideal sample message size. Feel free to go maximum till 150 : smaller values will create lot of outbound connections. { yield return(outgoingList); outgoingList.Clear(); sizeOfListItems = 0; } outgoingList.Add(denormalizedRecord); sizeOfListItems += 1; } //for Event hub binding fork -- end //other output bindings else if (sizeOfListItems + sizeOfDenormalizedRecord > MAXTRANSMISSIONSIZE + 20) { yield return(outgoingList); outgoingList.Clear(); sizeOfListItems = 0; } outgoingList.Add(denormalizedRecord); sizeOfListItems += sizeOfDenormalizedRecord; } } } } if (sizeOfListItems > 0) { yield return(outgoingList); } } finally { ListPool <DenormalizedRecord> .Free(outgoingList); } }
static IEnumerable <List <DenormalizedRecord> > denormalizedRecords(string newClientContent, Binder errorRecordBinder, ILogger log) { var outgoingList = ListPool <DenormalizedRecord> .Allocate(); outgoingList.Capacity = 450; var sizeOfListItems = 0; try { NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent); foreach (var record in logs.records) { float version = record.properties.Version; foreach (var outerFlow in record.properties.flows) { foreach (var innerFlow in outerFlow.flows) { foreach (var flowTuple in innerFlow.flowTuples) { var tuple = new NSGFlowLogTuple(flowTuple, version); var denormalizedRecord = new DenormalizedRecord( record.properties.Version, record.time, record.category, record.operationName, record.resourceId, outerFlow.rule, innerFlow.mac, tuple); var sizeOfDenormalizedRecord = denormalizedRecord.GetSizeOfJSONObject(); //new-code-start //if (sizeOfDenormalizedRecord > 120) // this will chunk below 256KB if (sizeOfListItems > 120) { yield return(outgoingList); outgoingList.Clear(); sizeOfListItems = 0; } //new-code-end /* * if (sizeOfListItems + sizeOfDenormalizedRecord > MAXTRANSMISSIONSIZE + 20) * { * yield return outgoingList; * outgoingList.Clear(); * sizeOfListItems = 0; * } */ outgoingList.Add(denormalizedRecord); //sizeOfListItems += sizeOfDenormalizedRecord; sizeOfListItems += 1; } } } } if (sizeOfListItems > 0) { yield return(outgoingList); } } finally { ListPool <DenormalizedRecord> .Free(outgoingList); } }