static System.Collections.Generic.IEnumerable <string> convertToCEF(string newClientContent, Binder errorRecordBinder, TraceWriter log)
{
// newClientContent is a json string with records
NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent);
string logIncomingJSON = Util.GetEnvironmentVariable("logIncomingJSON");
Boolean flag;
if (Boolean.TryParse(logIncomingJSON, out flag))
{
if (flag)
{
logErrorRecord(newClientContent, errorRecordBinder, log).Wait();
}
}
string cefRecordBase = "";
foreach (var record in logs.records)
{
float version = record.properties.Version;
cefRecordBase = record.MakeCEFTime();
cefRecordBase += "|Microsoft.Network";
cefRecordBase += "|NETWORKSECURITYGROUPS";
cefRecordBase += "|" + version.ToString("0.0");
cefRecordBase += "|" + record.category;
cefRecordBase += "|" + record.operationName;
cefRecordBase += "|1"; // severity is always 1
cefRecordBase += "|deviceExternalId=" + record.MakeDeviceExternalID();
foreach (var outerFlows in record.properties.flows)
{
// expectation is that there is only ever 1 item in record.properties.flows
string cefOuterFlowRecord = cefRecordBase;
cefOuterFlowRecord += String.Format(" cs1={0}", outerFlows.rule);
cefOuterFlowRecord += String.Format(" cs1Label=NSGRuleName");
foreach (var innerFlows in outerFlows.flows)
{
var cefInnerFlowRecord = cefOuterFlowRecord;
var firstFlowTupleEncountered = true;
foreach (var flowTuple in innerFlows.flowTuples)
{
var tuple = new NSGFlowLogTuple(flowTuple, version);
if (firstFlowTupleEncountered)
{
cefInnerFlowRecord += (tuple.GetDirection == "I" ? " dmac=" : " smac=") + innerFlows.MakeMAC();
firstFlowTupleEncountered = false;
}
yield return(cefInnerFlowRecord + " " + tuple.ToString());
}
}
}
}
}
public DenormalizedRecord(
float version,
string time,
string category,
string operationName,
string resourceId,
string nsgRuleName,
string mac,
NSGFlowLogTuple tuple
)
{
this.version = version;
this.time = time;
this.category = category;
this.operationName = operationName;
this.resourceId = resourceId;
this.nsgRuleName = nsgRuleName;
this.mac = mac;
this.startTime = tuple.startTime;
this.sourceAddress = tuple.sourceAddress;
this.destinationAddress = tuple.destinationAddress;
this.sourcePort = tuple.sourcePort;
this.destinationPort = tuple.destinationPort;
this.transportProtocol = tuple.transportProtocol;
this.deviceDirection = tuple.deviceDirection;
this.deviceAction = tuple.deviceAction;
if (this.version >= 2.0)
{
this.flowState = tuple.flowState;
this.packetsDtoS = tuple.packetsDtoS;
this.packetsStoD = tuple.packetsStoD;
this.bytesDtoS = tuple.bytesDtoS;
this.bytesStoD = tuple.bytesStoD;
}
}
/// <summary>
/// input newClientContent is a string representation of a json array of records, each of which is a nsg flow log hierarchy
/// output is a List of SplunkEventMessage, up to a max # of bytes or 450 elements
/// </summary>
/// <param name="newClientContent"></param>
/// <param name="errorRecordBinder"></param>
/// <param name="log"></param>
/// <returns></returns>
static IEnumerable <List <SplunkEventMessage> > denormalizedSplunkEvents(string newClientContent, Binder errorRecordBinder, ILogger log)
{
var outgoingSplunkList = ListPool <SplunkEventMessage> .Allocate();
outgoingSplunkList.Capacity = 450;
var sizeOfListItems = 0;
try
{
NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent);
foreach (var record in logs.records)
{
float version = record.properties.Version;
foreach (var outerFlow in record.properties.flows)
{
foreach (var innerFlow in outerFlow.flows)
{
foreach (var flowTuple in innerFlow.flowTuples)
{
var tuple = new NSGFlowLogTuple(flowTuple, version);
var denormalizedRecord = new DenormalizedRecord(
record.properties.Version,
record.time,
record.category,
record.operationName,
record.resourceId,
outerFlow.rule,
innerFlow.mac,
tuple);
var splunkEventMessage = new SplunkEventMessage(denormalizedRecord);
var sizeOfObject = splunkEventMessage.GetSizeOfObject();
if (sizeOfListItems + sizeOfObject > MAXTRANSMISSIONSIZE + 20 || outgoingSplunkList.Count == 450)
{
yield return(outgoingSplunkList);
outgoingSplunkList.Clear();
sizeOfListItems = 0;
}
outgoingSplunkList.Add(splunkEventMessage);
sizeOfListItems += sizeOfObject;
}
}
}
}
if (sizeOfListItems > 0)
{
yield return(outgoingSplunkList);
}
}
finally
{
ListPool <SplunkEventMessage> .Free(outgoingSplunkList);
}
}
private static DenormalizedRecord createDenormalizedRecord2V2()
{
NSGFlowLogTuple tuple = new NSGFlowLogTuple("1578673962,10.244.0.40,10.244.1.68,36098,25227,U,O,D,C,10,1300,9,600", 2.0f);
DenormalizedRecord record = new DenormalizedRecord(2.0f,
"2020-01-15T07:00:00.5173253Z",
"NetworkSecurityGroupFlowEvent",
"NetworkSecurityGroupFlowEvents",
"/SUBSCRIPTIONS/F087A016-314D-482C-93F1-88665DAFBA23/RESOURCEGROUPS/MC_MDRNWRK-DEV-AKS-RESOURCES_MDRNWRK-DEV-AKS_UKSOUTH/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/AKS-AGENTPOOL-14244569-NSG",
"DefaultRule_AllowVnetOutBound",
"000D3R5F1340",
tuple);
return(record);
}
static System.Collections.Generic.IEnumerable <string> convertToSplunk(string newClientContent, Binder errorRecordBinder, TraceWriter log)
{
//
// newClientContent looks like this:
//
// {
// "records":[
// {...},
// {...}
// ...
// ]
// }
//
NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent);
string logIncomingJSON = Util.GetEnvironmentVariable("logIncomingJSON");
Boolean flag;
if (Boolean.TryParse(logIncomingJSON, out flag))
{
if (flag)
{
logErrorRecord(newClientContent, errorRecordBinder, log).Wait();
}
}
var sbBase = new StringBuilder();
foreach (var record in logs.records)
{
float version = record.properties.Version;
sbBase.Clear();
sbBase.Append("{");
sbBase.Append(eqs("time", true)).Append(eqs(record.time));
sbBase.Append(eqs(true, "category")).Append(eqs(record.category));
sbBase.Append(eqs(true, "operationName")).Append(eqs(record.operationName));
sbBase.Append(eqs(true, "version")).Append(eqs(version.ToString("0.0")));
sbBase.Append(eqs(true, "deviceExtId")).Append(eqs(record.MakeDeviceExternalID()));
int count = 1;
var sbOuterFlowRecord = new StringBuilder();
foreach (var outerFlows in record.properties.flows)
{
sbOuterFlowRecord.Clear();
sbOuterFlowRecord.Append(sbBase.ToString());
sbOuterFlowRecord.Append(eqs(true, "flowOrder")).Append(eqs(count.ToString()));
sbOuterFlowRecord.Append(eqs(true, "nsgRuleName")).Append(eqs(outerFlows.rule));
var sbInnerFlowRecord = new StringBuilder();
foreach (var innerFlows in outerFlows.flows)
{
sbInnerFlowRecord.Clear();
sbInnerFlowRecord.Append(sbOuterFlowRecord.ToString());
var firstFlowTupleEncountered = true;
foreach (var flowTuple in innerFlows.flowTuples)
{
var tuple = new NSGFlowLogTuple(flowTuple, version);
if (firstFlowTupleEncountered)
{
sbInnerFlowRecord.Append((tuple.GetDirection == "I" ? eqs(true, "dmac") : eqs(true, "smac"))).Append(eqs(innerFlows.MakeMAC()));
firstFlowTupleEncountered = false;
}
yield return(sbInnerFlowRecord.ToString() + tuple.JsonSubString() + "}");
}
}
}
}
}
static IEnumerable <List <DenormalizedRecord> > denormalizedRecords(string newClientContent, Binder errorRecordBinder, ILogger log)
{
var outgoingList = ListPool <DenormalizedRecord> .Allocate();
outgoingList.Capacity = 450;
var sizeOfListItems = 0;
try
{
NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent);
foreach (var record in logs.records)
{
float version = record.properties.Version;
foreach (var outerFlow in record.properties.flows)
{
foreach (var innerFlow in outerFlow.flows)
{
foreach (var flowTuple in innerFlow.flowTuples)
{
var tuple = new NSGFlowLogTuple(flowTuple, version);
var denormalizedRecord = new DenormalizedRecord(
record.properties.Version,
record.time,
record.category,
record.operationName,
record.resourceId,
outerFlow.rule,
innerFlow.mac,
tuple);
var sizeOfDenormalizedRecord = denormalizedRecord.GetSizeOfJSONObject();
//for Event hub binding fork -- start
// Event hub basic message size is 256KB and the 'if' statement below ensures that list does not exceed size this size for Eventhub
string outputBinding = Util.GetEnvironmentVariable("outputBinding");
if (outputBinding == "eventhub")
{
if (sizeOfListItems > 120) // this will chunk below 256KB : this is ideal sample message size. Feel free to go maximum till 150 : smaller values will create lot of outbound connections.
{
yield return(outgoingList);
outgoingList.Clear();
sizeOfListItems = 0;
}
outgoingList.Add(denormalizedRecord);
sizeOfListItems += 1;
}
//for Event hub binding fork -- end
//other output bindings
else if (sizeOfListItems + sizeOfDenormalizedRecord > MAXTRANSMISSIONSIZE + 20)
{
yield return(outgoingList);
outgoingList.Clear();
sizeOfListItems = 0;
}
outgoingList.Add(denormalizedRecord);
sizeOfListItems += sizeOfDenormalizedRecord;
}
}
}
}
if (sizeOfListItems > 0)
{
yield return(outgoingList);
}
}
finally
{
ListPool <DenormalizedRecord> .Free(outgoingList);
}
}
static IEnumerable <List <DenormalizedRecord> > denormalizedRecords(string newClientContent, Binder errorRecordBinder, ILogger log)
{
var outgoingList = ListPool <DenormalizedRecord> .Allocate();
outgoingList.Capacity = 450;
var sizeOfListItems = 0;
try
{
NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent);
foreach (var record in logs.records)
{
float version = record.properties.Version;
foreach (var outerFlow in record.properties.flows)
{
foreach (var innerFlow in outerFlow.flows)
{
foreach (var flowTuple in innerFlow.flowTuples)
{
var tuple = new NSGFlowLogTuple(flowTuple, version);
var denormalizedRecord = new DenormalizedRecord(
record.properties.Version,
record.time,
record.category,
record.operationName,
record.resourceId,
outerFlow.rule,
innerFlow.mac,
tuple);
var sizeOfDenormalizedRecord = denormalizedRecord.GetSizeOfJSONObject();
//new-code-start
//if (sizeOfDenormalizedRecord > 120) // this will chunk below 256KB
if (sizeOfListItems > 120)
{
yield return(outgoingList);
outgoingList.Clear();
sizeOfListItems = 0;
}
//new-code-end
/*
* if (sizeOfListItems + sizeOfDenormalizedRecord > MAXTRANSMISSIONSIZE + 20)
* {
* yield return outgoingList;
* outgoingList.Clear();
* sizeOfListItems = 0;
* }
*/
outgoingList.Add(denormalizedRecord);
//sizeOfListItems += sizeOfDenormalizedRecord;
sizeOfListItems += 1;
}
}
}
}
if (sizeOfListItems > 0)
{
yield return(outgoingList);
}
}
finally
{
ListPool <DenormalizedRecord> .Free(outgoingList);
}
}
