/// <summary> /// Insert/Update alert records /// </summary> /// <param name="features"></param> /// <returns></returns> public void InsertAlerts(IEnumerable <FlowFeature> features) { StringBuilder sbSql = null; try { string flowType, srcMAC, dstMAC, protocol, timestamp; //// long srcIP, dstIP; string srcIP, dstIP; foreach (FlowFeature f in features) { sbSql = new StringBuilder(); flowType = Flow2.GetFlowTypeName(f.Type); // srcIP = ConvertIpToLong(f.SrcIP); srcIP = ConvertIpToString(f.SrcIP); srcMAC = ConvertMACAddressToString(f.SrcMAC); // dstIP = ConvertIpToLong(f.DestIP); dstIP = ConvertIpToString(f.DestIP); dstMAC = ConvertMACAddressToString(f.DestMAC); protocol = GetProtocolName(f.Protocol); // timestamp = DateTime.UtcNow.ToString("o");//o means: 2008-06-15T21:15:07.0000000 //Just getting the milliseconds percision timestamp = f.DetectionTimeStamp.ToString("o"); timestamp = timestamp.Substring(0, timestamp.Length - 5); if (Double.IsInfinity(f.PPS)) { f.PPS = 0.0d; } sbSql.Append(INSERT_SQL); sbSql.AppendFormat(" (\"{0}\", \"{1}\", {2}, {3}, {4}, {5}, {6}, {7}, \"{8}\", \"{9}\", {10}, \"{11}\", \"{12}\", {13}, \"{14}\", \"{15}\", {16}, {17}, {18}, {19}, {20}, {21}, {22}, {23}, {24}, {25}, {26});", f.LoggerIp, //f.DetectionTimeStamp.ToString("o"), timestamp, f.APL, f.PV, f.PX, f.PPS, f.FPS, f.DPL, flowType, srcIP, f.SrcPort, srcMAC, dstIP, f.DestPort, dstMAC, protocol, f.AB, f.TBT, f.BS, f.PS, f.NNP, f.NSP, f.PSP, f.Duration, f.AIT, f.IOPR, f.Reconnect ); sbSql.AppendLine(); //System.Diagnostics.Debug.WriteLine(sbSql.ToString()); //using the connection inside the loop is intentional to introduce a bit //of delay so that we don't have overlaping timestamps for the alerts using (MySqlConnection _dbConnection = new MySqlConnection(_connectionString)) { _dbConnection.Open(); using (MySqlCommand cmd = new MySqlCommand(sbSql.ToString(), _dbConnection)) { cmd.ExecuteNonQuery(); } _dbConnection.Close(); } } }//end for catch (MySqlException sqlEx) { //throw sqlEx; System.Diagnostics.Debug.WriteLine(sqlEx.ToString()); } catch (Exception ex) { //throw ex; System.Diagnostics.Debug.WriteLine(ex.ToString()); } finally { ; } sbSql.Clear(); sbSql = null; }
public PrimaryConnection2(Flow2 f) { this.SrcIP = f.SrcIP; this.DestIP = f.SrcIP; }
/// <summary> /// Generate flow features for the flow.The current flow is in a TimeWindow already. /// </summary> /// <returns></returns> public ConversationFeature GenerateConversationFeatuesInTimeWindow() { ConversationFeature conversationFlowFeature = new ConversationFeature(); conversationFlowFeature.SrcIP = this._conn.SrcIP.GetAddressBytes(); conversationFlowFeature.DestIP = this._conn.DestIP.GetAddressBytes(); conversationFlowFeature.SrcMAC = this._flows[0].GetPackets().First().SrcMAC; conversationFlowFeature.DestMAC = this._flows[0].GetPackets().First().DestMAC; conversationFlowFeature.Protocol = (int)(this.Protocol); conversationFlowFeature.sizeofFirstPkt = _flows[0].GetPackets().First().PayloadLength; //flowFeature.DetectionTimeStamp = _pktList[0].ReceivingTime; //total number of packet transmitted in the conversation for (int i = 0; i < _flows.Count; i++) { conversationFlowFeature.PPC += _flows[i].NbOfPkt; } uint totalPayloadLength = 0; uint PayloadLengthSmaller146 = 0; uint PayloadLengthLarger146 = 0; int pktEx = 0; int less = 0; int large = 0; int noIncomingPkts = 0; int noOutgoingPkt = 0; uint noIncomingByte = 0; uint noOutgoingByte = 0; for (int j = 0; j < _flows.Count; j++) { for (int i = 0; i < _flows[j].NbOfPkt; i++) { totalPayloadLength += _flows[j].GetPackets().ElementAt(i).PayloadLength; pktEx++; if (_flows[j].GetPackets().ElementAt(i).PayloadLength < 146) { PayloadLengthSmaller146 += _flows[j].GetPackets().ElementAt(i).PayloadLength; less++; } else if (_flows[j].GetPackets().ElementAt(i).PayloadLength > 146) { PayloadLengthLarger146 += _flows[j].GetPackets().ElementAt(i).PayloadLength; large++; } if (_flows[j].GetPackets().ElementAt(i).SrcEnd.Address.GetAddressBytes() == _conn.SrcIP.GetAddressBytes()) { noIncomingPkts++; noIncomingByte += _flows[j].GetPackets().ElementAt(i).PayloadLength; } else { noOutgoingPkt++; noOutgoingByte += _flows[j].GetPackets().ElementAt(i).PayloadLength; } } } conversationFlowFeature.NPS146 = PayloadLengthSmaller146; conversationFlowFeature.NPL146 = PayloadLengthLarger146; conversationFlowFeature.PPL146 = (double)PayloadLengthLarger146 / (double)large; conversationFlowFeature.PPS146 = (double)PayloadLengthSmaller146 / (double)less; conversationFlowFeature.BPC = totalPayloadLength; conversationFlowFeature.ALP = (double)totalPayloadLength / (double)pktEx; conversationFlowFeature.differences = Math.Abs(noIncomingPkts - noOutgoingPkt); conversationFlowFeature.byteDifferences = (uint)Math.Abs(noIncomingByte - noOutgoingByte); conversationFlowFeature.ratioPkts = (double)conversationFlowFeature.differences / (double)conversationFlowFeature.PPC; conversationFlowFeature.ratioBytes = (double)conversationFlowFeature.byteDifferences / (double)conversationFlowFeature.BPC; conversationFlowFeature.Type = Flow2.GetLabelIndex(this._flows[0].GetPackets().First().SrcMAC); double temp = (double)noIncomingPkts / (double)noOutgoingPkt; conversationFlowFeature.ratio1 = (double)temp + ((double)1 / (double)temp); double temp1 = (double)noIncomingByte / (double)noOutgoingByte; conversationFlowFeature.ratio2 = (double)temp1 + ((double)1 / (double)temp1); double temp2 = (double)conversationFlowFeature.BPC / (double)conversationFlowFeature.PPC; conversationFlowFeature.ratio3 = (double)temp2 + ((double)1 / (double)temp2); double temp3 = (double)noIncomingByte / (double)noIncomingPkts; double temp4 = (double)noOutgoingByte / (double)noOutgoingPkt; conversationFlowFeature.averrageDifference = (uint)Math.Abs(temp3 - temp4); _flows.Clear(); return(conversationFlowFeature); }
public ConversationPair(Flow2 flw) { this.SrcIP = flw.Connection.SrcEnd.Address; this.DestIP = flw.Connection.DestEnd.Address; this.Protocol = flw.Connection.Protocol; }
public FDSApplicationContext() : base() { try { dao = new MySqlDao(); Flow2.SetLabelTable(dao.GetFlowLabels()); } catch (Exception ex) { MessageBox.Show(ex.ToString()); MessageBox.Show("Please check DB connection, Exitting"); Application.Exit(); } //reading the default signature file string xmlSig = default(string); try { System.Console.WriteLine("the path is:" + Properties.Settings.Default.SignatureFile); using (StreamReader sr = new StreamReader(Properties.Settings.Default.SignatureFile)) { xmlSig = sr.ReadToEnd(); } System.Diagnostics.Debug.WriteLine("Deserializing Signature File: " + Properties.Settings.Default.SignatureFile); _tree = TestDecisionTree.XmlDeserialize(xmlSig); } catch (Exception ex) { MessageBox.Show("Problem with Signature File"); //terminate the thread, just return return; } _openTrainingSetDlg = new OpenFileDialog(); _openTrainingSetDlg.CheckFileExists = true; _openTrainingSetDlg.CheckPathExists = true; _openOfflineDetectionDlg = new OpenFileDialog(); _openOfflineDetectionDlg.CheckFileExists = true; _openOfflineDetectionDlg.CheckPathExists = true; _trainingPercentage = Properties.Settings.Default.TrainingPercentage; _dataServicePort = Properties.Settings.Default.DataSvcPort; _malFlowDetector = new MalFlowDetectMon(); ///TODO: We need to make this timewindow configrable _malFlowDetector.TimeWindow = new TimeSpan(0, 0, Properties.Settings.Default.TimeWindow); //_malFlowDetector.TimeWindow = new TimeSpan(0, 0, 1); _malFlowDetector.DetectionEvent += new NetworkDetectionEventHandler(ProcessNetworkEvent); _malFlowDetector.ErrorEvent += new PMErrorEventHandler(DisplayErrorEvent); _malFlowDetector.Start(); _dbThread = CreateDatabaseThread(); _dbThread.Start(); _components = new System.ComponentModel.Container(); _contextMenu = CreateContextMenu(); _notifyIcon = CreateSystrayIcon("Flow Detector Server", true); _notifyIcon.ContextMenu = _contextMenu; this.ThreadExit += new EventHandler(this.SystrayIcon_ApplicationExit); this.ThreadExit += new EventHandler(this.Application_Exit); }
/// <summary> /// Thread for training the decisition tree /// </summary> private void TrainingThread() { System.Console.WriteLine("Inside the training!!!!!!!!!!!!"); List <FlowFeature> trainingFeatures; // List<FlowFeature> trainingFeatures; //MySqlDao dao = null; try { //dao = new MySqlDao(); // set data table //Flow.SetLabelTable(dao.GetFlowLabels()); string traceFile = _openTrainingSetDlg.FileName; System.Console.WriteLine("the training file is:" + traceFile); if (File.Exists(traceFile) == false) { throw new FileNotFoundException("Trace file " + traceFile + " doesn't exist."); } NMParser parser = new NMParser(); IEnumerable <FlowFeature> allFeatures = GetFlowFeaturesFromTraceFile(parser, traceFile, -1); trainingFeatures = allFeatures.ToList(); List <Attribute> attributes = new List <Attribute>(); attributes.Add(new NumericalAttribute("PX")); attributes.Add(new NumericalAttribute("APL")); attributes.Add(new NumericalAttribute("PV")); attributes.Add(new NumericalAttribute("DPL")); attributes.Add(new NumericalAttribute("PPS")); attributes.Add(new IdSymbolicAttribute("Protocol", new List <string>() { "TCP", "UDP", "Mixed" })); attributes.Add(new NumericalAttribute("FPS")); attributes.Add(new NumericalAttribute("AB")); attributes.Add(new NumericalAttribute("TBT")); attributes.Add(new NumericalAttribute("BS")); attributes.Add(new NumericalAttribute("PS")); attributes.Add(new NumericalAttribute("NNP")); attributes.Add(new NumericalAttribute("NSP")); attributes.Add(new NumericalAttribute("PSP")); attributes.Add(new NumericalAttribute("Duration")); attributes.Add(new NumericalAttribute("AIT")); attributes.Add(new NumericalAttribute("IOPR")); attributes.Add(new NumericalAttribute("Reconnect")); attributes.Add(new IdSymbolicAttribute("Type", Flow2.GetFlowTypeNames())); // System.Diagnostics.Debug.WriteLine("TrainingThread1"); AttributeSet attrSet = new AttributeSet(attributes); ItemSet itemSet = new ItemSet(attrSet); Dictionary <int, int> maliciousFlowCounter = new Dictionary <int, int>(); int value; foreach (FlowFeature feature in trainingFeatures) { List <AttributeValue> attrVals = new List <AttributeValue>(); attrVals.Add(new KnownNumericalValue(feature.PX)); attrVals.Add(new KnownNumericalValue(feature.APL)); attrVals.Add(new KnownNumericalValue(feature.PV)); attrVals.Add(new KnownNumericalValue(feature.DPL)); attrVals.Add(new KnownNumericalValue(feature.PPS)); attrVals.Add(new KnownSymbolicValue((int)feature.Protocol)); attrVals.Add(new KnownNumericalValue(feature.FPS)); attrVals.Add(new KnownNumericalValue(feature.AB)); attrVals.Add(new KnownNumericalValue(feature.TBT)); attrVals.Add(new KnownNumericalValue(feature.BS)); attrVals.Add(new KnownNumericalValue(feature.PS)); attrVals.Add(new KnownNumericalValue(feature.NNP)); attrVals.Add(new KnownNumericalValue(feature.NSP)); attrVals.Add(new KnownNumericalValue(feature.PSP)); attrVals.Add(new KnownNumericalValue(feature.Duration)); attrVals.Add(new KnownNumericalValue(feature.AIT)); attrVals.Add(new KnownNumericalValue(feature.IOPR)); attrVals.Add(new KnownNumericalValue(feature.Reconnect)); attrVals.Add(new KnownSymbolicValue(feature.Type)); // System.Diagnostics.Debug.WriteLine("TrainingThread2"); // attrVals.Add(new ((DateTime)feature.DetectionTimeStamp)); Item it = new Item(attrVals.ToArray()); if (feature.Type > 0) // if the flow is not normal, count { if (!maliciousFlowCounter.TryGetValue(feature.Type, out value)) { maliciousFlowCounter.Add(feature.Type, 1); } else { maliciousFlowCounter[feature.Type]++; } } itemSet.Add(it); } foreach (int index in maliciousFlowCounter.Keys) { System.Diagnostics.Debug.WriteLine("Number of Malicious Flows for type: " + Flow2.GetFlowTypeName(index) + " is: " + maliciousFlowCounter[index].ToString()); } SymbolicAttribute goalAttribute = attrSet.FindByName("Type") as SymbolicAttribute; List <Attribute> testAttributes = new List <Attribute>(); testAttributes.Add(attrSet.FindByName("PX")); testAttributes.Add(attrSet.FindByName("APL")); testAttributes.Add(attrSet.FindByName("PV")); testAttributes.Add(attrSet.FindByName("DPL")); testAttributes.Add(attrSet.FindByName("PPS")); testAttributes.Add(attrSet.FindByName("Protocol")); testAttributes.Add(attrSet.FindByName("FPS")); // testAttributes.Add(attrSet.FindByName("Type")); testAttributes.Add(attrSet.FindByName("AB")); testAttributes.Add(attrSet.FindByName("TBT")); testAttributes.Add(attrSet.FindByName("BS")); testAttributes.Add(attrSet.FindByName("PS")); testAttributes.Add(attrSet.FindByName("NNP")); testAttributes.Add(attrSet.FindByName("NSP")); testAttributes.Add(attrSet.FindByName("PSP")); testAttributes.Add(attrSet.FindByName("Duration")); testAttributes.Add(attrSet.FindByName("AIT")); testAttributes.Add(attrSet.FindByName("IOPR")); testAttributes.Add(attrSet.FindByName("Reconnect")); // System.Diagnostics.Debug.WriteLine("TrainingThread3"); SimpleDecisionTreeBuilder builder = new SimpleDecisionTreeBuilder( /// create tree hear! new WeightedItemSet(itemSet), new AttributeSet(testAttributes), goalAttribute); builder.ScoreThreshold = 0.0001d; // 0.0001 * itemSet.Size(); System.Diagnostics.Debug.WriteLine("DT ScoreThreshold is " + builder.ScoreThreshold.ToString()); LearningDecisionTree dt = builder.Build(); TestDecisionTree tdt = new TestDecisionTree(dt); StoreDecisionTree(tdt); } catch (ThreadInterruptedException) { ; } catch (ThreadAbortException) { ; } catch (Exception ex) { MessageBox.Show(ex.ToString(), "Error"); } finally { menuTraining.Text = Properties.Resources.StartTrainingText; } }
/// <summary> /// Generate flow features for the flow.The current flow is in a TimeWindow already. /// </summary> /// <returns></returns> public FlowFeature GenerateFeatuesInTimeWindow() { // // System.Console.WriteLine(" inside GenerateFeatuesInTimeWindow "); if (_pktList.Count < 3) { return(null); } //collecting same length packet SortedList <uint, int> samePktLength = new SortedList <uint, int>(); int pktEx = 0; long totalPayloadLength = 0; long totalTCPUDPPayloadLength = 0; int pktExTCPUDP = 0; int noNullPacket = 0; int noSmallPacket = 0; double timeWindow = new TimeSpan(0, 0, Properties.Settings.Default.TimeWindow).TotalSeconds; FlowFeature flowFeature = new FlowFeature(); flowFeature.SrcIP = this._conn.SrcEnd.Address.GetAddressBytes(); flowFeature.SrcPort = this._conn.SrcEnd.Port; flowFeature.DestIP = this._conn.DestEnd.Address.GetAddressBytes(); flowFeature.DestPort = this._conn.DestEnd.Port; //getting the MAC address of the first packet for the flow //we assume that the MAC address is consistant per flow flowFeature.SrcMAC = this._pktList[0].SrcMAC; flowFeature.DestMAC = this._pktList[0].DestMAC; flowFeature.Protocol = (int)(this.Protocol); flowFeature.Reconnect = this.reconnect; flowFeature.FPS = (double)_pktList[0].PayloadLength; /// should be revised // flowFeature.tcpFlag = 0; flowFeature.DetectionTimeStamp = _pktList[0].ReceivingTime; DateTime[] interArrivalTime = new DateTime[_pktList.Count]; uint noIncommingPackets = 0; uint noOutgoingPackets = 0; for (int i = 0; i < _pktList.Count; i++) { totalPayloadLength += _pktList[i].PayloadLength; pktEx++; interArrivalTime[i] = _pktList[i].ReceivingTime; if (_pktList[i].SrcEnd.Address == _conn.SrcEnd.Address) { noIncommingPackets++; } else { noOutgoingPackets++; } Packet2.Protocol temp = new Packet2.Protocol(); if (temp != Packet2.Protocol.Mixed) //to capture TCP/UDP flows { //either TCP/UDP totalTCPUDPPayloadLength += _pktList[i].PayloadLength; pktExTCPUDP++; } if (_pktList[i].PayloadLength == 0) { noNullPacket += 1; } if (63 < _pktList[i].PayloadLength && _pktList[i].PayloadLength < 399) // counting th enumber of small packets { noSmallPacket += 1; } if (samePktLength.ContainsKey(_pktList[i].PayloadLength)) { samePktLength[_pktList[i].PayloadLength] += 1; } else { samePktLength.Add(_pktList[i].PayloadLength, 1); } } //// ////// // flowFeature.Type = Flow2.GetLabelIndex(this._conn.SrcEnd.Address.GetAddressBytes()); flowFeature.Type = Flow2.GetLabelIndex_ME(this._conn, flowFeature.SrcMAC, flowFeature.DestMAC); // if (flowFeature.Type == 3 || flowFeature.Type == 4 || flowFeature.Type == 5) // System.Diagnostics.Debug.WriteLine(flowFeature.ToString()); flowFeature.PX = (double)pktEx; flowFeature.APL = (double)(totalPayloadLength) / (double)pktEx; // flowFeature.PX = (double)pktEx; flowFeature.PV = CalVariance(samePktLength, flowFeature.APL); flowFeature.DPL = (double)(samePktLength.Count) / pktEx; flowFeature.AB = (double)totalTCPUDPPayloadLength / (double)pktExTCPUDP; flowFeature.TBT = totalPayloadLength; flowFeature.BS = (double)totalPayloadLength / (double)timeWindow; flowFeature.PS = (double)pktEx / (double)timeWindow; flowFeature.NNP = noNullPacket; flowFeature.NSP = noSmallPacket; flowFeature.PSP = ((double)noSmallPacket / (double)pktEx) * 100; flowFeature.Duration = (this.StopTime - this.StartTime).TotalSeconds; flowFeature.AIT = AverageIntervalTime(interArrivalTime); if (this.NoBackwardPackets != 0) { flowFeature.IOPR = (double)this.NbOfPkt / (double)this.NoBackwardPackets; } else { flowFeature.IOPR = -1; } // flowFeature.SDNP = Variance(samePktLength, flowFeature.APL); flowFeature.PPS = pktEx < 2 ? 0.0 : (double)(pktEx) * 1000.0d / (_pktList.Last().ReceivingTime - _pktList.First().ReceivingTime).TotalSeconds; if (Double.IsInfinity(flowFeature.PPS)) { flowFeature.PPS = 0.0d; } // we do reseat all the calculation to be used by next flow because the function calling this is in a loop samePktLength.Clear(); samePktLength = null; // _pktList.Clear(); return(flowFeature); }