/// <summary>
/// input newClientContent is a string representation of a json array of records, each of which is a nsg flow log hierarchy
/// output is a List of SplunkEventMessage, up to a max # of bytes or 450 elements
/// </summary>
/// <param name="newClientContent"></param>
/// <param name="errorRecordBinder"></param>
/// <param name="log"></param>
/// <returns></returns>
static IEnumerable <List <SplunkEventMessage> > denormalizedSplunkEvents(string newClientContent, Binder errorRecordBinder, ILogger log)
{
var outgoingSplunkList = ListPool <SplunkEventMessage> .Allocate();
outgoingSplunkList.Capacity = 450;
var sizeOfListItems = 0;
try
{
NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent);
foreach (var record in logs.records)
{
float version = record.properties.Version;
foreach (var outerFlow in record.properties.flows)
{
foreach (var innerFlow in outerFlow.flows)
{
foreach (var flowTuple in innerFlow.flowTuples)
{
var tuple = new NSGFlowLogTuple(flowTuple, version);
var denormalizedRecord = new DenormalizedRecord(
record.properties.Version,
record.time,
record.category,
record.operationName,
record.resourceId,
outerFlow.rule,
innerFlow.mac,
tuple);
var splunkEventMessage = new SplunkEventMessage(denormalizedRecord);
var sizeOfObject = splunkEventMessage.GetSizeOfObject();
if (sizeOfListItems + sizeOfObject > MAXTRANSMISSIONSIZE + 20 || outgoingSplunkList.Count == 450)
{
yield return(outgoingSplunkList);
outgoingSplunkList.Clear();
sizeOfListItems = 0;
}
outgoingSplunkList.Add(splunkEventMessage);
sizeOfListItems += sizeOfObject;
}
}
}
}
if (sizeOfListItems > 0)
{
yield return(outgoingSplunkList);
}
}
finally
{
ListPool <SplunkEventMessage> .Free(outgoingSplunkList);
}
}
public Source(DenormalizedRecord denormalizedRecord)
{
this.address = denormalizedRecord.sourceAddress;
this.ip = denormalizedRecord.sourceAddress;
this.port = Convert.ToInt64(denormalizedRecord.sourcePort);
this.packets = Convert.ToInt64(denormalizedRecord.packetsStoD);
this.bytes = Convert.ToInt64(denormalizedRecord.bytesStoD);
}
public Destination(DenormalizedRecord denormalizedRecord)
{
this.address = denormalizedRecord.destinationAddress;
this.ip = denormalizedRecord.destinationAddress;
this.port = Convert.ToInt64(denormalizedRecord.destinationPort);
this.packets = Convert.ToInt64(denormalizedRecord.packetsDtoS);
this.bytes = Convert.ToInt64(denormalizedRecord.bytesDtoS);
}
public Network(DenormalizedRecord denormalizedRecord)
{
this.transport = (denormalizedRecord.transportProtocol == "U" ? "udp" : "tcp");
this.direction = (denormalizedRecord.deviceDirection == "I" ? "inbound" : "outbound");
this.protocol = "transport";
this.bytes = Convert.ToInt64(denormalizedRecord.bytesStoD) + Convert.ToInt64(denormalizedRecord.bytesDtoS);
this.packets = Convert.ToInt64(denormalizedRecord.packetsStoD) + Convert.ToInt64(denormalizedRecord.packetsDtoS);
this.flowstate = denormalizedRecord.flowState;
}
public void denormalizedRecordJsonTest()
{
DenormalizedRecord denormalizedRecordV2 = EcsTest.createDenormalizedRecordV2();
var outgoingJson = JsonConvert.SerializeObject(denormalizedRecordV2, new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore
});
String expected = "{\"time\":\"2020-01-15T07:00:00.5173253Z\",\"category\":\"NetworkSecurityGroupFlowEvent\",\"operationName\":\"NetworkSecurityGroupFlowEvents\",\"resourceId\":\"/SUBSCRIPTIONS/F087A016-314D-482C-93F1-88665DAFBA23/RESOURCEGROUPS/MC_MDRNWRK-DEV-AKS-RESOURCES_MDRNWRK-DEV-AKS_UKSOUTH/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/AKS-AGENTPOOL-14244569-NSG\",\"version\":2.0,\"nsgRuleName\":\"DefaultRule_AllowVnetOutBound\",\"mac\":\"000D3R5F1340\",\"startTime\":\"1578673962\",\"sourceAddress\":\"10.244.0.40\",\"destinationAddress\":\"10.244.1.68\",\"sourcePort\":\"36098\",\"destinationPort\":\"25227\",\"transportProtocol\":\"T\",\"deviceDirection\":\"I\",\"deviceAction\":\"A\",\"flowState\":\"E\",\"packetsStoD\":\"3\",\"bytesStoD\":\"206\",\"packetsDtoS\":\"2\",\"bytesDtoS\":\"140\"}";
Assert.Equal(expected, outgoingJson);
}
public EcsAll(DenormalizedRecord denormalizedRecord)
{
this.timestamp = denormalizedRecord.time;
this.agent = new Agent("AzureNetworkWatcherNSGFlowLogsConnector");
this.rule = new Rule(denormalizedRecord.nsgRuleName);
this.ecs = new Ecs("1.0.0");
this.type = "ecs-flowlogs-azurensg";
this.client = new Client(denormalizedRecord.mac);
this.ecsevent = new EcsEvent(denormalizedRecord);
this.resource = new Resource(denormalizedRecord.resourceId);
this.source = new Source(denormalizedRecord);
this.destination = new Destination(denormalizedRecord);
this.network = new Network(denormalizedRecord);
}
private static DenormalizedRecord createDenormalizedRecord2V2()
{
NSGFlowLogTuple tuple = new NSGFlowLogTuple("1578673962,10.244.0.40,10.244.1.68,36098,25227,U,O,D,C,10,1300,9,600", 2.0f);
DenormalizedRecord record = new DenormalizedRecord(2.0f,
"2020-01-15T07:00:00.5173253Z",
"NetworkSecurityGroupFlowEvent",
"NetworkSecurityGroupFlowEvents",
"/SUBSCRIPTIONS/F087A016-314D-482C-93F1-88665DAFBA23/RESOURCEGROUPS/MC_MDRNWRK-DEV-AKS-RESOURCES_MDRNWRK-DEV-AKS_UKSOUTH/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/AKS-AGENTPOOL-14244569-NSG",
"DefaultRule_AllowVnetOutBound",
"000D3R5F1340",
tuple);
return(record);
}
public EcsEvent(DenormalizedRecord denormalizedRecord)
{
this.category = denormalizedRecord.category;
this.action = denormalizedRecord.operationName;
this.outcome = (denormalizedRecord.deviceAction == "A" ? "allowed" : "denied");
DateTimeOffset dateTimeOffset = DateTimeOffset.FromUnixTimeSeconds(Convert.ToInt64(denormalizedRecord.startTime));
this.start = dateTimeOffset.UtcDateTime.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ");
this.dataset = "nsg.access";
DateTime ingestedUtcNow = DateTime.UtcNow;
this.ingested = ingestedUtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ");
}
public void denormalizedRecordToEcsTest()
{
DenormalizedRecord denormalizedRecordV2 = EcsTest.createDenormalizedRecordV2();
EcsAll ecsAll = EcsFactory.createEcsAll(denormalizedRecordV2);
var outgoingJson = JsonConvert.SerializeObject(ecsAll, new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
Formatting = Newtonsoft.Json.Formatting.Indented,
});
output.WriteLine(outgoingJson);
Assert.Equal(denormalizedRecordV2.time, ecsAll.@timestamp);
Assert.Equal("AzureNetworkWatcherNSGFlowLogsConnector", ecsAll.agent.name);
Assert.Equal(denormalizedRecordV2.nsgRuleName, ecsAll.rule.name);
Assert.Equal("1.0.0", ecsAll.ecs.version);
Assert.Equal(denormalizedRecordV2.mac, ecsAll.client.mac);
Assert.Equal(denormalizedRecordV2.category, ecsAll.ecsevent.category);
Assert.Equal(denormalizedRecordV2.operationName, ecsAll.ecsevent.action);
Assert.Equal("allowed", ecsAll.ecsevent.outcome);
Assert.Equal("nsg.access", ecsAll.ecsevent.dataset);
Assert.Equal("2020-01-10T16:32:42.0000000Z", ecsAll.ecsevent.start);
Assert.Equal(denormalizedRecordV2.resourceId, ecsAll.resource.id);
Assert.Equal("F087A016-314D-482C-93F1-88665DAFBA23", ecsAll.resource.subscription);
Assert.Equal("AKS-AGENTPOOL-14244569-NSG", ecsAll.resource.nsg);
Assert.Equal("10.244.0.40", ecsAll.source.address);
Assert.Equal("10.244.0.40", ecsAll.source.ip);
Assert.Equal("10.244.1.68", ecsAll.destination.address);
Assert.Equal("10.244.1.68", ecsAll.destination.ip);
Assert.Equal(36098, ecsAll.source.port);
Assert.Equal(25227, ecsAll.destination.port);
Assert.Equal(3, ecsAll.source.packets);
Assert.Equal(2, ecsAll.destination.packets);
Assert.Equal(206, ecsAll.source.bytes);
Assert.Equal(140, ecsAll.destination.bytes);
Assert.Equal("tcp", ecsAll.network.transport);
Assert.Equal("inbound", ecsAll.network.direction);
Assert.Equal("transport", ecsAll.network.protocol);
Assert.Equal(5, ecsAll.network.packets);
Assert.Equal(346, ecsAll.network.bytes);
Assert.Equal("E", ecsAll.network.flowstate);
}
public SplunkEventMessage(DenormalizedRecord splunkEvent)
{
sourcetype = "amdl:nsg:flowlogs";
time = unixTime(splunkEvent.time);
@event = splunkEvent;
}
static IEnumerable <List <DenormalizedRecord> > denormalizedRecords(string newClientContent, Binder errorRecordBinder, ILogger log)
{
var outgoingList = ListPool <DenormalizedRecord> .Allocate();
outgoingList.Capacity = 450;
var sizeOfListItems = 0;
try
{
NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent);
foreach (var record in logs.records)
{
float version = record.properties.Version;
foreach (var outerFlow in record.properties.flows)
{
foreach (var innerFlow in outerFlow.flows)
{
foreach (var flowTuple in innerFlow.flowTuples)
{
var tuple = new NSGFlowLogTuple(flowTuple, version);
var denormalizedRecord = new DenormalizedRecord(
record.properties.Version,
record.time,
record.category,
record.operationName,
record.resourceId,
outerFlow.rule,
innerFlow.mac,
tuple);
var sizeOfDenormalizedRecord = denormalizedRecord.GetSizeOfJSONObject();
//for Event hub binding fork -- start
// Event hub basic message size is 256KB and the 'if' statement below ensures that list does not exceed size this size for Eventhub
string outputBinding = Util.GetEnvironmentVariable("outputBinding");
if (outputBinding == "eventhub")
{
if (sizeOfListItems > 120) // this will chunk below 256KB : this is ideal sample message size. Feel free to go maximum till 150 : smaller values will create lot of outbound connections.
{
yield return(outgoingList);
outgoingList.Clear();
sizeOfListItems = 0;
}
outgoingList.Add(denormalizedRecord);
sizeOfListItems += 1;
}
//for Event hub binding fork -- end
//other output bindings
else if (sizeOfListItems + sizeOfDenormalizedRecord > MAXTRANSMISSIONSIZE + 20)
{
yield return(outgoingList);
outgoingList.Clear();
sizeOfListItems = 0;
}
outgoingList.Add(denormalizedRecord);
sizeOfListItems += sizeOfDenormalizedRecord;
}
}
}
}
if (sizeOfListItems > 0)
{
yield return(outgoingList);
}
}
finally
{
ListPool <DenormalizedRecord> .Free(outgoingList);
}
}
public SplunkEventMessage(DenormalizedRecord splunkEvent)
{
sourcetype = "amdl:nsg:flowlogs";
@event = splunkEvent;
}
public static EcsAll createEcsAll(DenormalizedRecord denormalizedRecord)
{
EcsAll ecsAll = new EcsAll(denormalizedRecord);
return(ecsAll);
}
static IEnumerable <List <DenormalizedRecord> > denormalizedRecords(string newClientContent, Binder errorRecordBinder, ILogger log)
{
var outgoingList = ListPool <DenormalizedRecord> .Allocate();
outgoingList.Capacity = 450;
var sizeOfListItems = 0;
try
{
NSGFlowLogRecords logs = JsonConvert.DeserializeObject <NSGFlowLogRecords>(newClientContent);
foreach (var record in logs.records)
{
float version = record.properties.Version;
foreach (var outerFlow in record.properties.flows)
{
foreach (var innerFlow in outerFlow.flows)
{
foreach (var flowTuple in innerFlow.flowTuples)
{
var tuple = new NSGFlowLogTuple(flowTuple, version);
var denormalizedRecord = new DenormalizedRecord(
record.properties.Version,
record.time,
record.category,
record.operationName,
record.resourceId,
outerFlow.rule,
innerFlow.mac,
tuple);
var sizeOfDenormalizedRecord = denormalizedRecord.GetSizeOfJSONObject();
//new-code-start
//if (sizeOfDenormalizedRecord > 120) // this will chunk below 256KB
if (sizeOfListItems > 120)
{
yield return(outgoingList);
outgoingList.Clear();
sizeOfListItems = 0;
}
//new-code-end
/*
* if (sizeOfListItems + sizeOfDenormalizedRecord > MAXTRANSMISSIONSIZE + 20)
* {
* yield return outgoingList;
* outgoingList.Clear();
* sizeOfListItems = 0;
* }
*/
outgoingList.Add(denormalizedRecord);
//sizeOfListItems += sizeOfDenormalizedRecord;
sizeOfListItems += 1;
}
}
}
}
if (sizeOfListItems > 0)
{
yield return(outgoingList);
}
}
finally
{
ListPool <DenormalizedRecord> .Free(outgoingList);
}
}