/// <summary>
/// Insert/Update alert records
/// </summary>
/// <param name="features"></param>
/// <returns></returns>
public void InsertAlerts(IEnumerable <FlowFeature> features)
{
StringBuilder sbSql = null;
try
{
string flowType, srcMAC, dstMAC, protocol, timestamp;
//// long srcIP, dstIP;
string srcIP, dstIP;
foreach (FlowFeature f in features)
{
sbSql = new StringBuilder();
flowType = Flow2.GetFlowTypeName(f.Type);
// srcIP = ConvertIpToLong(f.SrcIP);
srcIP = ConvertIpToString(f.SrcIP);
srcMAC = ConvertMACAddressToString(f.SrcMAC);
// dstIP = ConvertIpToLong(f.DestIP);
dstIP = ConvertIpToString(f.DestIP);
dstMAC = ConvertMACAddressToString(f.DestMAC);
protocol = GetProtocolName(f.Protocol);
// timestamp = DateTime.UtcNow.ToString("o");//o means: 2008-06-15T21:15:07.0000000
//Just getting the milliseconds percision
timestamp = f.DetectionTimeStamp.ToString("o");
timestamp = timestamp.Substring(0, timestamp.Length - 5);
if (Double.IsInfinity(f.PPS))
{
f.PPS = 0.0d;
}
sbSql.Append(INSERT_SQL);
sbSql.AppendFormat(" (\"{0}\", \"{1}\", {2}, {3}, {4}, {5}, {6}, {7}, \"{8}\", \"{9}\", {10}, \"{11}\", \"{12}\", {13}, \"{14}\", \"{15}\", {16}, {17}, {18}, {19}, {20}, {21}, {22}, {23}, {24}, {25}, {26});",
f.LoggerIp,
//f.DetectionTimeStamp.ToString("o"),
timestamp,
f.APL,
f.PV,
f.PX,
f.PPS,
f.FPS,
f.DPL,
flowType,
srcIP,
f.SrcPort,
srcMAC,
dstIP,
f.DestPort,
dstMAC,
protocol,
f.AB,
f.TBT,
f.BS,
f.PS,
f.NNP,
f.NSP,
f.PSP,
f.Duration,
f.AIT,
f.IOPR,
f.Reconnect
);
sbSql.AppendLine();
//System.Diagnostics.Debug.WriteLine(sbSql.ToString());
//using the connection inside the loop is intentional to introduce a bit
//of delay so that we don't have overlaping timestamps for the alerts
using (MySqlConnection _dbConnection = new MySqlConnection(_connectionString))
{
_dbConnection.Open();
using (MySqlCommand cmd = new MySqlCommand(sbSql.ToString(), _dbConnection))
{
cmd.ExecuteNonQuery();
}
_dbConnection.Close();
}
}
}//end for
catch (MySqlException sqlEx)
{
//throw sqlEx;
System.Diagnostics.Debug.WriteLine(sqlEx.ToString());
}
catch (Exception ex)
{
//throw ex;
System.Diagnostics.Debug.WriteLine(ex.ToString());
}
finally
{
;
}
sbSql.Clear();
sbSql = null;
}
public PrimaryConnection2(Flow2 f)
{
this.SrcIP = f.SrcIP;
this.DestIP = f.SrcIP;
}
/// <summary>
/// Generate flow features for the flow.The current flow is in a TimeWindow already.
/// </summary>
/// <returns></returns>
public ConversationFeature GenerateConversationFeatuesInTimeWindow()
{
ConversationFeature conversationFlowFeature = new ConversationFeature();
conversationFlowFeature.SrcIP = this._conn.SrcIP.GetAddressBytes();
conversationFlowFeature.DestIP = this._conn.DestIP.GetAddressBytes();
conversationFlowFeature.SrcMAC = this._flows[0].GetPackets().First().SrcMAC;
conversationFlowFeature.DestMAC = this._flows[0].GetPackets().First().DestMAC;
conversationFlowFeature.Protocol = (int)(this.Protocol);
conversationFlowFeature.sizeofFirstPkt = _flows[0].GetPackets().First().PayloadLength;
//flowFeature.DetectionTimeStamp = _pktList[0].ReceivingTime;
//total number of packet transmitted in the conversation
for (int i = 0; i < _flows.Count; i++)
{
conversationFlowFeature.PPC += _flows[i].NbOfPkt;
}
uint totalPayloadLength = 0;
uint PayloadLengthSmaller146 = 0;
uint PayloadLengthLarger146 = 0;
int pktEx = 0;
int less = 0;
int large = 0;
int noIncomingPkts = 0;
int noOutgoingPkt = 0;
uint noIncomingByte = 0;
uint noOutgoingByte = 0;
for (int j = 0; j < _flows.Count; j++)
{
for (int i = 0; i < _flows[j].NbOfPkt; i++)
{
totalPayloadLength += _flows[j].GetPackets().ElementAt(i).PayloadLength;
pktEx++;
if (_flows[j].GetPackets().ElementAt(i).PayloadLength < 146)
{
PayloadLengthSmaller146 += _flows[j].GetPackets().ElementAt(i).PayloadLength;
less++;
}
else if (_flows[j].GetPackets().ElementAt(i).PayloadLength > 146)
{
PayloadLengthLarger146 += _flows[j].GetPackets().ElementAt(i).PayloadLength;
large++;
}
if (_flows[j].GetPackets().ElementAt(i).SrcEnd.Address.GetAddressBytes() == _conn.SrcIP.GetAddressBytes())
{
noIncomingPkts++;
noIncomingByte += _flows[j].GetPackets().ElementAt(i).PayloadLength;
}
else
{
noOutgoingPkt++;
noOutgoingByte += _flows[j].GetPackets().ElementAt(i).PayloadLength;
}
}
}
conversationFlowFeature.NPS146 = PayloadLengthSmaller146;
conversationFlowFeature.NPL146 = PayloadLengthLarger146;
conversationFlowFeature.PPL146 = (double)PayloadLengthLarger146 / (double)large;
conversationFlowFeature.PPS146 = (double)PayloadLengthSmaller146 / (double)less;
conversationFlowFeature.BPC = totalPayloadLength;
conversationFlowFeature.ALP = (double)totalPayloadLength / (double)pktEx;
conversationFlowFeature.differences = Math.Abs(noIncomingPkts - noOutgoingPkt);
conversationFlowFeature.byteDifferences = (uint)Math.Abs(noIncomingByte - noOutgoingByte);
conversationFlowFeature.ratioPkts = (double)conversationFlowFeature.differences / (double)conversationFlowFeature.PPC;
conversationFlowFeature.ratioBytes = (double)conversationFlowFeature.byteDifferences / (double)conversationFlowFeature.BPC;
conversationFlowFeature.Type = Flow2.GetLabelIndex(this._flows[0].GetPackets().First().SrcMAC);
double temp = (double)noIncomingPkts / (double)noOutgoingPkt;
conversationFlowFeature.ratio1 = (double)temp + ((double)1 / (double)temp);
double temp1 = (double)noIncomingByte / (double)noOutgoingByte;
conversationFlowFeature.ratio2 = (double)temp1 + ((double)1 / (double)temp1);
double temp2 = (double)conversationFlowFeature.BPC / (double)conversationFlowFeature.PPC;
conversationFlowFeature.ratio3 = (double)temp2 + ((double)1 / (double)temp2);
double temp3 = (double)noIncomingByte / (double)noIncomingPkts;
double temp4 = (double)noOutgoingByte / (double)noOutgoingPkt;
conversationFlowFeature.averrageDifference = (uint)Math.Abs(temp3 - temp4);
_flows.Clear();
return(conversationFlowFeature);
}
public ConversationPair(Flow2 flw)
{
this.SrcIP = flw.Connection.SrcEnd.Address;
this.DestIP = flw.Connection.DestEnd.Address;
this.Protocol = flw.Connection.Protocol;
}
public FDSApplicationContext()
: base()
{
try
{
dao = new MySqlDao();
Flow2.SetLabelTable(dao.GetFlowLabels());
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
MessageBox.Show("Please check DB connection, Exitting");
Application.Exit();
}
//reading the default signature file
string xmlSig = default(string);
try
{
System.Console.WriteLine("the path is:" + Properties.Settings.Default.SignatureFile);
using (StreamReader sr = new StreamReader(Properties.Settings.Default.SignatureFile))
{
xmlSig = sr.ReadToEnd();
}
System.Diagnostics.Debug.WriteLine("Deserializing Signature File: " + Properties.Settings.Default.SignatureFile);
_tree = TestDecisionTree.XmlDeserialize(xmlSig);
}
catch (Exception ex)
{
MessageBox.Show("Problem with Signature File");
//terminate the thread, just return
return;
}
_openTrainingSetDlg = new OpenFileDialog();
_openTrainingSetDlg.CheckFileExists = true;
_openTrainingSetDlg.CheckPathExists = true;
_openOfflineDetectionDlg = new OpenFileDialog();
_openOfflineDetectionDlg.CheckFileExists = true;
_openOfflineDetectionDlg.CheckPathExists = true;
_trainingPercentage = Properties.Settings.Default.TrainingPercentage;
_dataServicePort = Properties.Settings.Default.DataSvcPort;
_malFlowDetector = new MalFlowDetectMon();
///TODO: We need to make this timewindow configrable
_malFlowDetector.TimeWindow = new TimeSpan(0, 0, Properties.Settings.Default.TimeWindow);
//_malFlowDetector.TimeWindow = new TimeSpan(0, 0, 1);
_malFlowDetector.DetectionEvent += new NetworkDetectionEventHandler(ProcessNetworkEvent);
_malFlowDetector.ErrorEvent += new PMErrorEventHandler(DisplayErrorEvent);
_malFlowDetector.Start();
_dbThread = CreateDatabaseThread();
_dbThread.Start();
_components = new System.ComponentModel.Container();
_contextMenu = CreateContextMenu();
_notifyIcon = CreateSystrayIcon("Flow Detector Server", true);
_notifyIcon.ContextMenu = _contextMenu;
this.ThreadExit += new EventHandler(this.SystrayIcon_ApplicationExit);
this.ThreadExit += new EventHandler(this.Application_Exit);
}
/// <summary>
/// Thread for training the decisition tree
/// </summary>
private void TrainingThread()
{
System.Console.WriteLine("Inside the training!!!!!!!!!!!!");
List <FlowFeature> trainingFeatures;
// List<FlowFeature> trainingFeatures;
//MySqlDao dao = null;
try
{
//dao = new MySqlDao();
// set data table
//Flow.SetLabelTable(dao.GetFlowLabels());
string traceFile = _openTrainingSetDlg.FileName;
System.Console.WriteLine("the training file is:" + traceFile);
if (File.Exists(traceFile) == false)
{
throw new FileNotFoundException("Trace file " + traceFile + " doesn't exist.");
}
NMParser parser = new NMParser();
IEnumerable <FlowFeature> allFeatures = GetFlowFeaturesFromTraceFile(parser, traceFile, -1);
trainingFeatures = allFeatures.ToList();
List <Attribute> attributes = new List <Attribute>();
attributes.Add(new NumericalAttribute("PX"));
attributes.Add(new NumericalAttribute("APL"));
attributes.Add(new NumericalAttribute("PV"));
attributes.Add(new NumericalAttribute("DPL"));
attributes.Add(new NumericalAttribute("PPS"));
attributes.Add(new IdSymbolicAttribute("Protocol", new List <string>()
{
"TCP", "UDP", "Mixed"
}));
attributes.Add(new NumericalAttribute("FPS"));
attributes.Add(new NumericalAttribute("AB"));
attributes.Add(new NumericalAttribute("TBT"));
attributes.Add(new NumericalAttribute("BS"));
attributes.Add(new NumericalAttribute("PS"));
attributes.Add(new NumericalAttribute("NNP"));
attributes.Add(new NumericalAttribute("NSP"));
attributes.Add(new NumericalAttribute("PSP"));
attributes.Add(new NumericalAttribute("Duration"));
attributes.Add(new NumericalAttribute("AIT"));
attributes.Add(new NumericalAttribute("IOPR"));
attributes.Add(new NumericalAttribute("Reconnect"));
attributes.Add(new IdSymbolicAttribute("Type", Flow2.GetFlowTypeNames()));
// System.Diagnostics.Debug.WriteLine("TrainingThread1");
AttributeSet attrSet = new AttributeSet(attributes);
ItemSet itemSet = new ItemSet(attrSet);
Dictionary <int, int> maliciousFlowCounter = new Dictionary <int, int>();
int value;
foreach (FlowFeature feature in trainingFeatures)
{
List <AttributeValue> attrVals = new List <AttributeValue>();
attrVals.Add(new KnownNumericalValue(feature.PX));
attrVals.Add(new KnownNumericalValue(feature.APL));
attrVals.Add(new KnownNumericalValue(feature.PV));
attrVals.Add(new KnownNumericalValue(feature.DPL));
attrVals.Add(new KnownNumericalValue(feature.PPS));
attrVals.Add(new KnownSymbolicValue((int)feature.Protocol));
attrVals.Add(new KnownNumericalValue(feature.FPS));
attrVals.Add(new KnownNumericalValue(feature.AB));
attrVals.Add(new KnownNumericalValue(feature.TBT));
attrVals.Add(new KnownNumericalValue(feature.BS));
attrVals.Add(new KnownNumericalValue(feature.PS));
attrVals.Add(new KnownNumericalValue(feature.NNP));
attrVals.Add(new KnownNumericalValue(feature.NSP));
attrVals.Add(new KnownNumericalValue(feature.PSP));
attrVals.Add(new KnownNumericalValue(feature.Duration));
attrVals.Add(new KnownNumericalValue(feature.AIT));
attrVals.Add(new KnownNumericalValue(feature.IOPR));
attrVals.Add(new KnownNumericalValue(feature.Reconnect));
attrVals.Add(new KnownSymbolicValue(feature.Type));
// System.Diagnostics.Debug.WriteLine("TrainingThread2");
// attrVals.Add(new ((DateTime)feature.DetectionTimeStamp));
Item it = new Item(attrVals.ToArray());
if (feature.Type > 0) // if the flow is not normal, count
{
if (!maliciousFlowCounter.TryGetValue(feature.Type, out value))
{
maliciousFlowCounter.Add(feature.Type, 1);
}
else
{
maliciousFlowCounter[feature.Type]++;
}
}
itemSet.Add(it);
}
foreach (int index in maliciousFlowCounter.Keys)
{
System.Diagnostics.Debug.WriteLine("Number of Malicious Flows for type: " + Flow2.GetFlowTypeName(index) + " is: " + maliciousFlowCounter[index].ToString());
}
SymbolicAttribute goalAttribute = attrSet.FindByName("Type") as SymbolicAttribute;
List <Attribute> testAttributes = new List <Attribute>();
testAttributes.Add(attrSet.FindByName("PX"));
testAttributes.Add(attrSet.FindByName("APL"));
testAttributes.Add(attrSet.FindByName("PV"));
testAttributes.Add(attrSet.FindByName("DPL"));
testAttributes.Add(attrSet.FindByName("PPS"));
testAttributes.Add(attrSet.FindByName("Protocol"));
testAttributes.Add(attrSet.FindByName("FPS"));
// testAttributes.Add(attrSet.FindByName("Type"));
testAttributes.Add(attrSet.FindByName("AB"));
testAttributes.Add(attrSet.FindByName("TBT"));
testAttributes.Add(attrSet.FindByName("BS"));
testAttributes.Add(attrSet.FindByName("PS"));
testAttributes.Add(attrSet.FindByName("NNP"));
testAttributes.Add(attrSet.FindByName("NSP"));
testAttributes.Add(attrSet.FindByName("PSP"));
testAttributes.Add(attrSet.FindByName("Duration"));
testAttributes.Add(attrSet.FindByName("AIT"));
testAttributes.Add(attrSet.FindByName("IOPR"));
testAttributes.Add(attrSet.FindByName("Reconnect"));
// System.Diagnostics.Debug.WriteLine("TrainingThread3");
SimpleDecisionTreeBuilder builder = new SimpleDecisionTreeBuilder( /// create tree hear!
new WeightedItemSet(itemSet),
new AttributeSet(testAttributes),
goalAttribute);
builder.ScoreThreshold = 0.0001d; // 0.0001 * itemSet.Size();
System.Diagnostics.Debug.WriteLine("DT ScoreThreshold is " + builder.ScoreThreshold.ToString());
LearningDecisionTree dt = builder.Build();
TestDecisionTree tdt = new TestDecisionTree(dt);
StoreDecisionTree(tdt);
}
catch (ThreadInterruptedException)
{
;
}
catch (ThreadAbortException)
{
;
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString(), "Error");
}
finally
{
menuTraining.Text = Properties.Resources.StartTrainingText;
}
}
/// <summary>
/// Generate flow features for the flow.The current flow is in a TimeWindow already.
/// </summary>
/// <returns></returns>
public FlowFeature GenerateFeatuesInTimeWindow()
{
//
// System.Console.WriteLine(" inside GenerateFeatuesInTimeWindow ");
if (_pktList.Count < 3)
{
return(null);
}
//collecting same length packet
SortedList <uint, int> samePktLength = new SortedList <uint, int>();
int pktEx = 0;
long totalPayloadLength = 0;
long totalTCPUDPPayloadLength = 0;
int pktExTCPUDP = 0;
int noNullPacket = 0;
int noSmallPacket = 0;
double timeWindow = new TimeSpan(0, 0, Properties.Settings.Default.TimeWindow).TotalSeconds;
FlowFeature flowFeature = new FlowFeature();
flowFeature.SrcIP = this._conn.SrcEnd.Address.GetAddressBytes();
flowFeature.SrcPort = this._conn.SrcEnd.Port;
flowFeature.DestIP = this._conn.DestEnd.Address.GetAddressBytes();
flowFeature.DestPort = this._conn.DestEnd.Port;
//getting the MAC address of the first packet for the flow
//we assume that the MAC address is consistant per flow
flowFeature.SrcMAC = this._pktList[0].SrcMAC;
flowFeature.DestMAC = this._pktList[0].DestMAC;
flowFeature.Protocol = (int)(this.Protocol);
flowFeature.Reconnect = this.reconnect;
flowFeature.FPS = (double)_pktList[0].PayloadLength; /// should be revised
// flowFeature.tcpFlag = 0;
flowFeature.DetectionTimeStamp = _pktList[0].ReceivingTime;
DateTime[] interArrivalTime = new DateTime[_pktList.Count];
uint noIncommingPackets = 0;
uint noOutgoingPackets = 0;
for (int i = 0; i < _pktList.Count; i++)
{
totalPayloadLength += _pktList[i].PayloadLength;
pktEx++;
interArrivalTime[i] = _pktList[i].ReceivingTime;
if (_pktList[i].SrcEnd.Address == _conn.SrcEnd.Address)
{
noIncommingPackets++;
}
else
{
noOutgoingPackets++;
}
Packet2.Protocol temp = new Packet2.Protocol();
if (temp != Packet2.Protocol.Mixed) //to capture TCP/UDP flows
{ //either TCP/UDP
totalTCPUDPPayloadLength += _pktList[i].PayloadLength;
pktExTCPUDP++;
}
if (_pktList[i].PayloadLength == 0)
{
noNullPacket += 1;
}
if (63 < _pktList[i].PayloadLength && _pktList[i].PayloadLength < 399) // counting th enumber of small packets
{
noSmallPacket += 1;
}
if (samePktLength.ContainsKey(_pktList[i].PayloadLength))
{
samePktLength[_pktList[i].PayloadLength] += 1;
}
else
{
samePktLength.Add(_pktList[i].PayloadLength, 1);
}
}
////
//////
// flowFeature.Type = Flow2.GetLabelIndex(this._conn.SrcEnd.Address.GetAddressBytes());
flowFeature.Type = Flow2.GetLabelIndex_ME(this._conn, flowFeature.SrcMAC, flowFeature.DestMAC);
// if (flowFeature.Type == 3 || flowFeature.Type == 4 || flowFeature.Type == 5)
// System.Diagnostics.Debug.WriteLine(flowFeature.ToString());
flowFeature.PX = (double)pktEx;
flowFeature.APL = (double)(totalPayloadLength) / (double)pktEx;
// flowFeature.PX = (double)pktEx;
flowFeature.PV = CalVariance(samePktLength, flowFeature.APL);
flowFeature.DPL = (double)(samePktLength.Count) / pktEx;
flowFeature.AB = (double)totalTCPUDPPayloadLength / (double)pktExTCPUDP;
flowFeature.TBT = totalPayloadLength;
flowFeature.BS = (double)totalPayloadLength / (double)timeWindow;
flowFeature.PS = (double)pktEx / (double)timeWindow;
flowFeature.NNP = noNullPacket;
flowFeature.NSP = noSmallPacket;
flowFeature.PSP = ((double)noSmallPacket / (double)pktEx) * 100;
flowFeature.Duration = (this.StopTime - this.StartTime).TotalSeconds;
flowFeature.AIT = AverageIntervalTime(interArrivalTime);
if (this.NoBackwardPackets != 0)
{
flowFeature.IOPR = (double)this.NbOfPkt / (double)this.NoBackwardPackets;
}
else
{
flowFeature.IOPR = -1;
}
// flowFeature.SDNP = Variance(samePktLength, flowFeature.APL);
flowFeature.PPS = pktEx < 2 ?
0.0
: (double)(pktEx) * 1000.0d
/ (_pktList.Last().ReceivingTime - _pktList.First().ReceivingTime).TotalSeconds;
if (Double.IsInfinity(flowFeature.PPS))
{
flowFeature.PPS = 0.0d;
}
// we do reseat all the calculation to be used by next flow because the function calling this is in a loop
samePktLength.Clear();
samePktLength = null;
// _pktList.Clear();
return(flowFeature);
}
