/// <summary>
/// Extract FlowFeatures from a trace file. Once the extract number of flow features are obtained, it should stop
/// </summary>
/// <param name="nmc"></param>
/// <param name="traceFileName"></param>
/// <param name="requiredCount">if the number is less than 0, extract all flows from the trace file.</param>
/// <returns></returns>
private IEnumerable <FlowFeature> GetFlowFeaturesFromTraceFile(NMParser nmc, string traceFileName, int requiredCount)
{
List <FlowFeature> features = new List <FlowFeature>();
//The Network Flows are extract from the tracefile by the time window.
//The GetNetworkFlows() function uses yield return.
IEnumerable <Flow2> temp = nmc.GetNetworkFlows(traceFileName, _malFlowDetector.TimeWindow);
var rows = new List <String>();
foreach (Flow2 f in temp)
//foreach(Flow f in nmc.GetNetworkFlows())
{
FlowFeature feature = f.GenerateFeatuesInTimeWindow();
if (feature != null)
{
feature.LoggerIp = "Offline";
rows.Add(feature.featuresToString());
features.Add(feature);
}
if (features.Count >= requiredCount && requiredCount > 0)
{
break;
}
}
//using (System.IO.StreamWriter file = new System.IO.StreamWriter(@"C:\Users\ebiglarb\Desktop\signature.csv"))
//{
// RuntimeConfig config = new RuntimeConfig();
// String featuresNameList = config.featuresList();
// file.WriteLine(featuresNameList);
// foreach (string line in rows)
// {
// file.WriteLine(line);
// }
//}
return(features);
}
/// <summary>
/// Generate flow features for the flow.The current flow is in a TimeWindow already.
/// </summary>
/// <returns></returns>
public PairNodeFeature GenerateFeatuesInTimeWindow()
{
//collecting same length packet
SortedList <double, int> sameFlowLength = new SortedList <double, int>();
int flEX = 0; // number of flows created between two nodes
int pktEx = 0; // number of packets exchanged between a pair of nodes
double totalPayloadLength = 0;
long totalTCPUDPPayloadLength = 0;
int pktExTCPUDP = 0;
int noNullPacket = 0;
int noSmallPacket = 0;
double timeWindow = new TimeSpan(0, 0, Properties.Settings.Default.TimeWindow).TotalSeconds;
PairNodeFeature pairFeature = new PairNodeFeature();
pairFeature.SrcIP = this._pairConn.SrcIP.GetAddressBytes();
// pairFeature.SrcPort = this._conn.SrcEnd.Port;
pairFeature.DestIP = this._pairConn.DestIP.GetAddressBytes();
// pairFeature.DestPort = this._conn.DestEnd.Port;
//getting the MAC address of the first packet for the flow
//we assume that the MAC address is consistant per flow
pairFeature.SrcMAC = this._flowList[0].GetPackets().First().SrcMAC;
pairFeature.DestMAC = this._flowList[0].GetPackets().First().DestMAC;
DateTime[] interArrivalTime = new DateTime[_flowList.Count];
int noIncommingFlows = 0;
int noOutgoingFlows = 0;
uint NoOutgoingPkt = 0; // from dest to src
uint NoIncommingPkt = 0; // from node src to dest
for (int i = 0; i < _flowList.Count; i++)
{
double flowLength = 0;
FlowFeature flowFeature = _flowList[i].GenerateFeatuesInTimeWindow();
NoIncommingPkt += flowFeature.NoIncommingPkt;
flowLength = (double)_flowList[i].NbOfPkt * flowFeature.APL; //total payload leangth of that flow
totalPayloadLength += flowLength;
pktEx += _flowList[i].NbOfPkt;
interArrivalTime[i] = flowFeature.DetectionTimeStamp; // arrival time of each flow = recieving time of first packet in that flow
if (_flowList[i].SrcIP == _pairConn.SrcIP)
{
noIncommingFlows++;
}
else
{
noOutgoingFlows++;
}
if (sameFlowLength.ContainsKey(flowLength))
{
sameFlowLength[flowLength] += 1;
}
else
{
sameFlowLength.Add(flowLength, 1);
}
}
pairFeature.Type = GetLabelIndex(_flowList[0].SrcIP.GetAddressBytes());
pairFeature.NofFlows = _flowList.Count;
pairFeature.ConnectionDegree = pairFeature.NoIncommingPkt + pairFeature.NoOutgoingPkt + _flowList.Count;
// we do reseat all the calculation to be used by next flow because the function calling this is in a loop
sameFlowLength.Clear();
sameFlowLength = null;
_flowList.Clear();
return(pairFeature);
}
private void consumeFeature(FlowFeature feature)
{
try{
if (this._tree == null)
{
throw new Exception("No signature was found.");
}
//OnErrorEvent(new Exception("Detecting on " + features.Count + " flows"));
List <AttributeValue> attrVals = new List <AttributeValue>();
attrVals.Add(new KnownNumericalValue(feature.PX));
attrVals.Add(new KnownNumericalValue(feature.APL));
attrVals.Add(new KnownNumericalValue(feature.PV));
attrVals.Add(new KnownNumericalValue(feature.DPL));
attrVals.Add(new KnownNumericalValue(feature.PPS));
attrVals.Add(new KnownSymbolicValue(feature.Protocol));
attrVals.Add(new KnownNumericalValue(feature.FPS));
attrVals.Add(new KnownNumericalValue(feature.AB));
attrVals.Add(new KnownNumericalValue(feature.TBT));
attrVals.Add(new KnownNumericalValue(feature.BS));
attrVals.Add(new KnownNumericalValue(feature.PS));
attrVals.Add(new KnownNumericalValue(feature.NNP));
attrVals.Add(new KnownNumericalValue(feature.NSP));
attrVals.Add(new KnownNumericalValue(feature.PSP));
attrVals.Add(new KnownNumericalValue(feature.Duration));
attrVals.Add(new KnownNumericalValue(feature.AIT));
attrVals.Add(new KnownNumericalValue(feature.IOPR));
attrVals.Add(new KnownNumericalValue(feature.Reconnect));
// System.Diagnostics.Debug.WriteLine("consume feature 1");
//attrVals.Add(new KnownSymbolicValue((int)(feature.Type)));
Item it = new Item(attrVals.ToArray());
KnownSymbolicValue guessedVal = _tree.GuessGoalAttribute(it);
feature.Type = guessedVal.IntValue;
if (feature.Type != 0)
{
// feature.DetectionTimeStamp = DateTime.UtcNow;
//update malicious flows and total flows count
lock (_maliciousFlowsDetected)
{
_maliciousFlowsDetected.Add(feature);
}
}
feature = null;
it = null;
attrVals = null;
_totalNbFlowExamined += 1;
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString(), "Error");
}
}
/// <summary>
/// Generate flow features for the flow.The current flow is in a TimeWindow already.
/// </summary>
/// <returns></returns>
public FlowFeature GenerateFeatuesInTimeWindow()
{
//
// System.Console.WriteLine(" inside GenerateFeatuesInTimeWindow ");
if (_pktList.Count < 3)
{
return(null);
}
//collecting same length packet
SortedList <uint, int> samePktLength = new SortedList <uint, int>();
int pktEx = 0;
long totalPayloadLength = 0;
long totalTCPUDPPayloadLength = 0;
int pktExTCPUDP = 0;
int noNullPacket = 0;
int noSmallPacket = 0;
double timeWindow = new TimeSpan(0, 0, Properties.Settings.Default.TimeWindow).TotalSeconds;
FlowFeature flowFeature = new FlowFeature();
flowFeature.SrcIP = this._conn.SrcEnd.Address.GetAddressBytes();
flowFeature.SrcPort = this._conn.SrcEnd.Port;
flowFeature.DestIP = this._conn.DestEnd.Address.GetAddressBytes();
flowFeature.DestPort = this._conn.DestEnd.Port;
//getting the MAC address of the first packet for the flow
//we assume that the MAC address is consistant per flow
flowFeature.SrcMAC = this._pktList[0].SrcMAC;
flowFeature.DestMAC = this._pktList[0].DestMAC;
flowFeature.Protocol = (int)(this.Protocol);
flowFeature.Reconnect = this.reconnect;
flowFeature.FPS = (double)_pktList[0].PayloadLength; /// should be revised
// flowFeature.tcpFlag = 0;
flowFeature.DetectionTimeStamp = _pktList[0].ReceivingTime;
DateTime[] interArrivalTime = new DateTime[_pktList.Count];
uint noIncommingPackets = 0;
uint noOutgoingPackets = 0;
for (int i = 0; i < _pktList.Count; i++)
{
totalPayloadLength += _pktList[i].PayloadLength;
pktEx++;
interArrivalTime[i] = _pktList[i].ReceivingTime;
if (_pktList[i].SrcEnd.Address == _conn.SrcEnd.Address)
{
noIncommingPackets++;
}
else
{
noOutgoingPackets++;
}
Packet2.Protocol temp = new Packet2.Protocol();
if (temp != Packet2.Protocol.Mixed) //to capture TCP/UDP flows
{ //either TCP/UDP
totalTCPUDPPayloadLength += _pktList[i].PayloadLength;
pktExTCPUDP++;
}
if (_pktList[i].PayloadLength == 0)
{
noNullPacket += 1;
}
if (63 < _pktList[i].PayloadLength && _pktList[i].PayloadLength < 399) // counting th enumber of small packets
{
noSmallPacket += 1;
}
if (samePktLength.ContainsKey(_pktList[i].PayloadLength))
{
samePktLength[_pktList[i].PayloadLength] += 1;
}
else
{
samePktLength.Add(_pktList[i].PayloadLength, 1);
}
}
////
//////
// flowFeature.Type = Flow2.GetLabelIndex(this._conn.SrcEnd.Address.GetAddressBytes());
flowFeature.Type = Flow2.GetLabelIndex_ME(this._conn, flowFeature.SrcMAC, flowFeature.DestMAC);
// if (flowFeature.Type == 3 || flowFeature.Type == 4 || flowFeature.Type == 5)
// System.Diagnostics.Debug.WriteLine(flowFeature.ToString());
flowFeature.PX = (double)pktEx;
flowFeature.APL = (double)(totalPayloadLength) / (double)pktEx;
// flowFeature.PX = (double)pktEx;
flowFeature.PV = CalVariance(samePktLength, flowFeature.APL);
flowFeature.DPL = (double)(samePktLength.Count) / pktEx;
flowFeature.AB = (double)totalTCPUDPPayloadLength / (double)pktExTCPUDP;
flowFeature.TBT = totalPayloadLength;
flowFeature.BS = (double)totalPayloadLength / (double)timeWindow;
flowFeature.PS = (double)pktEx / (double)timeWindow;
flowFeature.NNP = noNullPacket;
flowFeature.NSP = noSmallPacket;
flowFeature.PSP = ((double)noSmallPacket / (double)pktEx) * 100;
flowFeature.Duration = (this.StopTime - this.StartTime).TotalSeconds;
flowFeature.AIT = AverageIntervalTime(interArrivalTime);
if (this.NoBackwardPackets != 0)
{
flowFeature.IOPR = (double)this.NbOfPkt / (double)this.NoBackwardPackets;
}
else
{
flowFeature.IOPR = -1;
}
// flowFeature.SDNP = Variance(samePktLength, flowFeature.APL);
flowFeature.PPS = pktEx < 2 ?
0.0
: (double)(pktEx) * 1000.0d
/ (_pktList.Last().ReceivingTime - _pktList.First().ReceivingTime).TotalSeconds;
if (Double.IsInfinity(flowFeature.PPS))
{
flowFeature.PPS = 0.0d;
}
// we do reseat all the calculation to be used by next flow because the function calling this is in a loop
samePktLength.Clear();
samePktLength = null;
// _pktList.Clear();
return(flowFeature);
}
