public static void printAllNodesToInfo(DFTGraph g) { foreach (DFTNode node in g.Vertices) { logger.Info(Utility.Graph.getNodeString(node)); } }
private VertexStats getVertexStatsFromNode(DFTGraph g, DFTNode n) { /* In Degree */ int inDeg = g.InDegree(n); /* Out Degree */ int outDeg = g.OutDegree(n); /* Size of Edge Data */ long sizeSumIncoming; long sizeSumOutgoing; computeStatsDataSize(g, n, out sizeSumIncoming, out sizeSumOutgoing); /* Centrality */ int centralityCount; int totalNodes = g.Vertices.Count(); double centralityRelative; computeStatsCentrality(g, n, totalNodes, out centralityCount, out centralityRelative); /* Closeness Centrality */ return(new VertexStats(n.node_id, n.nameFTR, inDeg, outDeg, sizeSumIncoming, sizeSumOutgoing, centralityCount, centralityRelative, totalNodes, n.nodeFeatures)); }
public void addStats(DFTGraph g, int time) { // iterate over all nodes in the (partial) graph foreach (DFTNode n in g.Vertices) { // if node is not yet recognized, create it with its initial stats if (!vertexStatCollection.ContainsKey(n.node_id)) { vertexStatCollection.Add(n.node_id, new VertexStatCollection(n.node_id, n.nameFTR, n.typeEnum, getVertexStatsFromNode(g, n), time)); nodes.Add(n); } else // if node is already recognized, obtain its stats collection // and update it with a new VertexStats object for this time instance { VertexStatCollection nodeStatCollection = vertexStatCollection[n.node_id]; if (nodeStatCollection.getVertexStatsForTime(time) != null) { logger.Fatal("Time Stats for Node ID " + n.node_id + " (" + n.nameFTR + ") already exist at time " + time); } else { nodeStatCollection.addVertexStatsForTime(time, getVertexStatsFromNode(g, n)); } } } }
private static void computeStatsDataSize(DFTGraph g, DFTNode n, out long sizeSumIncoming, out long sizeSumOutgoing) { // compute inData/outData (sum of weights of incoming/outgoing edges) // Console.WriteLine("Node: " + n.nameFTR + ", inDeg: " + g.InDegree(n).ToString()); List <DFTEdge> predEdges = g.getPredecessorEdges(n); List <DFTEdge> succEdges = g.getSuccessorEdges(n); sizeSumIncoming = 0; double sizeSumIncomingRel = 0; sizeSumOutgoing = 0; double sizeSumOutgoingRel = 0; int i = 1; foreach (DFTEdge e in predEdges) { // Console.WriteLine("Pred (" + i + "/" + predEdges.Count + "): " + e._name + ", size: " + e._size + ", Relative: " + e._inSizePercentage + ", (" + e.Source.nameFTR + "->" + e.Target.nameFTR + ")"); sizeSumIncoming += e._size; sizeSumIncomingRel += e._inSizePercentage; i++; } i = 0; foreach (DFTEdge e in succEdges) { // Console.WriteLine("Succ (" + i + "/" + succEdges.Count + "): " + e._name + ", size: " + e._size + ", Relative: " + e._outSizePercentage + ", (" + e.Source.nameFTR + "->" + e.Target.nameFTR + ")"); sizeSumOutgoing += e._size; sizeSumOutgoingRel += e._outSizePercentage; i++; } // Console.WriteLine("Sum Incoming: " + sizeSumIncoming + ", Percentage: " + sizeSumIncomingRel); // Console.WriteLine("Sum Outgoing: " + sizeSumOutgoing + ", Percentage: " + sizeSumIncomingRel); // Console.WriteLine("\n"); }
private void printMetrics(DFTGraph g) { foreach (DFTNode n in g.Vertices) { if (n.nameFTR.Contains("malware") || n.nameFTR.Contains("goodware")) { printMetrics(n); } } }
public static DFTGraph generateGraphFromCSV(String pathToEventLogCSV) { if (File.Exists(pathToEventLogCSV)) { //Console.WriteLine("Generate Graph: " + pathToEventLogCSV); DFTGraph graph = EventProcessor.generateGraphFromCSV(pathToEventLogCSV); return(graph); } else { throw new ArgumentException("Unable to generate Graph, No Event Log found at: " + pathToEventLogCSV); } }
public DFTGraph generateGraphFromLog() { logger.Info("Generating QDFG for Log"); try { DFTGraph g = Utility.Graph.generateGraphFromString(this.log.ToString()); return(g); } catch (Exception ex) { logger.Error(ex.Message); return(null); } }
private static List <DFTNode> getNodesByString(DFTGraph g, String value) { List <DFTNode> nodes = new List <DFTNode>(); foreach (DFTNode n in g.Vertices) { if (n.nameFTR.Contains(value)) { nodes.Add(n); } } return(nodes); }
private static void computeStatsCentrality(DFTGraph g, DFTNode n, int totalNodes, out int centralityCount, out double centralityRelative) { // Compute Centrality of Node // Centrality of node n: The number of nodes for which there exists a path p, such that p ends in node n, // in the directed graph, divided by the total number of nodes in the graph centralityCount = 0; foreach (DFTNode node in g.Vertices) { if (g.getConnectingEdges(node, n).Count > 0) { // Console.WriteLine("There exists a path from node " + node.nameFTR + ", to " + n.nameFTR); centralityCount++; } } centralityRelative = ((double)centralityCount / (double)totalNodes) * 100; // Console.WriteLine("Total Centrality Count for node " + n.nameFTR + ": " + centralityCount + ", Relative: " + centralityRelative + " (" + centralityCount + "/" + g.Vertices.Count() + ")"); }
/// <summary> /// Generate Dynamic Graph from Log File. Optionally generate Statistics and DataModel. /// </summary> /// <param name="logFile">Full Path to the Log File.</param> /// <param name="computeStats">true if Statistics and Model should be generated; otherwise, false.</param> internal void generateQDFG(String logFile, bool computeStats) { try { if (!Directory.Exists(Settings.outputDirectory + "\\" + Utility.IO.getFileNameFromPath(logFile))) { // Create Output folder DirectoryInfo di = Utility.IO.createOutputDirectory(Utility.IO.getFileNameFromPath(logFile)); if (di != null) { logger.Info(" --- Generating QDFG for Log <" + logFile + ">, Output Directory: \"" + di.Name + "\" --- "); // Generate & Output static Graph DFTGraph g = Utility.Graph.generateGraphFromCSV(logFile); String serializedGraph = g.serializeGraphML(); Utility.Graph.writeGraphToFile(serializedGraph, di.Name + "\\StaticGraph_" + Utility.IO.getFileNameFromPath(logFile) + ".graphml"); // Generate Dynamic Graph from Log processLogDynamic(logFile, di, computeStats); } else { logger.Fatal("Failed to Create output directory for log: " + Utility.IO.getFileNameFromPath(logFile)); } } else // skip previously processed log files { logger.Fatal("Already processed. (" + Settings.outputDirectory + "\\" + Utility.IO.getFileNameFromPath(logFile) + ")"); } } catch (Exception e) { logger.Fatal(logFile + ", failed to generate QDFG, " + e.Message); } }
public static List <DFTNode> getGoodwareNodes(DFTGraph g) { return(getNodesByString(g, "goodware")); }
public static List <DFTEdge> getAllOutgoingEdges(DFTGraph g, DFTNode n) { List <DFTEdge> outEdges = g.OutEdges(n).ToList(); return(outEdges); }
public static DFTNode getPythonNode(DFTGraph g) { DFTNode root = g.getNodeByName("P>python.exe"); return(root); }
public static DFTGraph generateGraphFromString(String log) { DFTGraph graph = EventProcessor.generateGraphFromString(log); return(graph); }
public static DFTGraph getReachableGraphPython(DFTGraph g) { DFTNode root = g.getNodeByName("P>python.exe"); return(g.getReachabilityGraphFWD(root)); }
public static List <DFTEdge> getAllIncomingEdges(DFTGraph g, DFTNode n) { List <DFTEdge> inEdges = g.InEdges(n).ToList(); return(inEdges); }
public static void writeGraph(DFTGraph g, String outputFile, bool dynamic, long start, long end, Dictionary <string, string> processedIDs) { XmlWriterSettings settings = new XmlWriterSettings(); settings.Indent = true; settings.Encoding = Encoding.UTF8; StringBuilder stringBuilder = new StringBuilder(); XmlWriter xmlWriter = XmlWriter.Create(stringBuilder, settings); xmlWriter.WriteStartDocument(); //xmlWriter.WriteRaw("\r\n<gexf xmlns=\"http://www.gexf.net/1.2draft\" version=\"1.2\">\r\n"); xmlWriter.WriteRaw("\r\n<gexf xmlns=\"http://www.gexf.net/1.2draft\" version=\"1.2\" xmlns:viz=\"http://www.gexf.net/1.2draft/viz\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.gexf.net/1.2draft http://www.gexf.net/1.2draft/gexf.xsd\">\r\n"); xmlWriter.WriteStartElement("graph"); if (dynamic) { xmlWriter.WriteAttributeString("mode", "dynamic"); xmlWriter.WriteAttributeString("timeformat", "double"); } else { xmlWriter.WriteAttributeString("mode", "static"); } xmlWriter.WriteAttributeString("defaultedgetype", "directed"); xmlWriter.WriteStartElement("attributes"); xmlWriter.WriteAttributeString("class", "node"); xmlWriter.WriteAttributeString("mode", "static"); xmlWriter.WriteStartElement("attribute"); xmlWriter.WriteAttributeString("id", "0"); xmlWriter.WriteAttributeString("title", "inDegree"); xmlWriter.WriteAttributeString("type", "long"); xmlWriter.WriteEndElement(); xmlWriter.WriteStartElement("attribute"); xmlWriter.WriteAttributeString("id", "1"); xmlWriter.WriteAttributeString("title", "outDegree"); xmlWriter.WriteAttributeString("type", "long"); xmlWriter.WriteEndElement(); xmlWriter.WriteEndElement(); if (dynamic) { xmlWriter.WriteStartElement("attributes"); xmlWriter.WriteAttributeString("class", "node"); xmlWriter.WriteAttributeString("mode", "dynamic"); xmlWriter.WriteStartElement("attribute"); xmlWriter.WriteAttributeString("id", "time_dynamic"); xmlWriter.WriteAttributeString("title", "time_dynamic"); xmlWriter.WriteAttributeString("type", "long"); xmlWriter.WriteEndElement(); xmlWriter.WriteEndElement(); } xmlWriter.WriteStartElement("nodes"); foreach (DFTNode n in g.Vertices) { xmlWriter.WriteStartElement("node"); xmlWriter.WriteAttributeString("id", n.node_id.ToString()); xmlWriter.WriteAttributeString("label", n.nameFTR); xmlWriter.WriteAttributeString("type", n._nodeType.ToString()); if (dynamic) { if (processedIDs.ContainsKey(n.nameFTR)) { xmlWriter.WriteAttributeString("start", processedIDs[n.nameFTR]); } else { logger.Fatal("Key " + n.nameFTR + " not found in time table"); } } xmlWriter.WriteStartElement("attvalues"); xmlWriter.WriteStartElement("attvalue"); xmlWriter.WriteAttributeString("for", "0"); xmlWriter.WriteAttributeString("value", g.InDegree(n).ToString()); xmlWriter.WriteEndElement(); xmlWriter.WriteStartElement("attvalue"); xmlWriter.WriteAttributeString("for", "1"); xmlWriter.WriteAttributeString("value", g.OutDegree(n).ToString()); xmlWriter.WriteEndElement(); if (dynamic) { if (processedIDs.ContainsKey(n.nameFTR)) { xmlWriter.WriteStartElement("attvalue"); xmlWriter.WriteAttributeString("for", "time_dynamic"); xmlWriter.WriteAttributeString("value", processedIDs[n.nameFTR]); xmlWriter.WriteAttributeString("start", start.ToString()); xmlWriter.WriteAttributeString("end", end.ToString()); xmlWriter.WriteEndElement(); } else { logger.Fatal("Key " + n.nameFTR + " not found in time table."); } } xmlWriter.WriteEndElement(); if (n._nodeType == DFTNodeType.SOCKET.ToString()) { xmlWriter.WriteRaw("\r\n<viz:color r=\"0\" g=\"0\" b=\"255\"></viz:color>\r\n"); } else if (n.nameFTR.Contains("malware")) { xmlWriter.WriteRaw("\r\n<viz:color r=\"255\" g=\"0\" b=\"0\"></viz:color>\r\n"); } else if (n.nameFTR.Contains("goodware")) { xmlWriter.WriteRaw("\r\n<viz:color r=\"0\" g=\"255\" b=\"0\"></viz:color>\r\n"); } else { xmlWriter.WriteRaw("\r\n<viz:color r=\"153\" g=\"153\" b=\"153\"></viz:color>\r\n"); } xmlWriter.WriteEndElement(); } xmlWriter.WriteEndElement(); xmlWriter.WriteStartElement("edges"); long edgeID = 0; foreach (DFTEdge e in g.Edges) { xmlWriter.WriteStartElement("edge"); xmlWriter.WriteAttributeString("id", edgeID.ToString()); xmlWriter.WriteAttributeString("source", e.Source.node_id.ToString()); xmlWriter.WriteAttributeString("target", e.Target.node_id.ToString()); xmlWriter.WriteAttributeString("label", e._name); xmlWriter.WriteAttributeString("Weight", e._size.ToString()); xmlWriter.WriteAttributeString("RelativeWeight", e.relSize.ToString()); //xmlWriter.WriteAttributeString("start", e.getMinTimestamp().ToString()); //xmlWriter.WriteAttributeString("end", e.getMaxTimestamp().ToString()); xmlWriter.WriteEndElement(); edgeID++; } xmlWriter.WriteEndElement(); xmlWriter.WriteEndElement(); xmlWriter.WriteRaw("\r\n</gexf>"); xmlWriter.WriteEndDocument(); xmlWriter.Flush(); xmlWriter.Close(); Utility.Graph.writeGraphToFile(stringBuilder.ToString(), outputFile); }
/// <summary> /// Actual Implementation of the DynamicLogProcessor. /// Generate Dynamic Graph from Log File. Optionally generate Statistics and DataModel. /// </summary> /// <param name="logPath">Full Path to the Log File.</param> /// <param name="diOutput">DirectoryInfo of the output Directory.</param> /// <param name="computeStats">true if Statistics and Model should be generated; otherwise, false.</param> private void processLogDynamic(String logPath, DirectoryInfo diOutput, bool computeStats) { // Reset the external EventProcessor (LibDFT) EventProcessor ep = new EventProcessor(); ep.freeResources(); logger.Info("Processing Log for Dynamics: " + logPath); // Init EventLog, Statistics & Metrics EventLog eventLog = new EventLog(logPath); QDFGStatCollection stats = new QDFGStatCollection(); CombinedMetrics metrics = new CombinedMetrics(); // Subpath for the Output Directory of this particular dynamic Log String outPath = diOutput.Name + "\\dynamicLog"; // Pre-process log file (Create Output Directories, Verify Log contains Nodes of interest) if (!preprocessLog(eventLog, logPath, outPath)) { return; } // Memory Check checkMemoryCount(logPath + ", Stage: init"); // Create Transformer for Event Log; Used to perform operations on EventLog EventLogTransformer transformer = new EventLogTransformer(eventLog); // Fix inconsistent time stamps & transform the absolute time representation into a relative one logger.Info("Fixing Time Info in Log: " + logPath); transformer.fixDates(); //eventLog.writeToFile(outPath + "\\sortedLog.txt"); // Memory Check checkMemoryCount(logPath + ", Stage: Fix Dates"); // Split Log into multiple Logs, each covering an increasing Interval (i.e. last Log is equivalent to original Log) logger.Info("Splitting Log additively: " + logPath); // List<EventLog> splitMergedLogs = transformer.splitAndMerge(Settings.eventSplitCount); List <EventLog> splitLogs = transformer.splitLogByTimeAdditive(Settings.timeStepMS); int numSamples = splitLogs.Count; // Number of Logs = Number of Intervals = Number of Samples for Features logger.Info("Split Done. Split into " + numSamples + " Logs."); // Memory Check checkMemoryCount(logPath + ", Stage: Split Logs"); // Only start advanced Processing of Interval Logs, if we have a sufficient Number of Samples if (numSamples >= Settings.MIN_SAMPLES) { Dictionary <string, string> processedIDs = new Dictionary <string, string>(); IEnumerable <DFTNode> verts = new List <DFTNode>(); DFTGraph lastGraph = new DFTGraph(); long start = -1; long end = splitLogs.Count + 10; bool monitoredFound = false; // For each Interval (partial) Log for (int i = 0; i < splitLogs.Count; i++) { // Memory Check checkMemoryCount(logPath + ", Stage: Partial Log " + i); if (monitoredFound || splitLogs[i].containsMonitoredProcess()) { logger.Debug("Contains Monitored " + i); if (start == -1) { start = i; monitoredFound = true; } logger.Info("Generating SubGraph " + i + "/" + splitLogs.Count); // TODO Investigate possible "Bug" in EventProcessor // At this point, somehow Graphs or Nodes/Edges that were generated in // previous calls of this method can leak state into this call. // This can be verified by calling generateGraphFromString // multiple times for different Logs and directly outputting the graph given by // the EventProcessor.generateGraphFromString(<someLog>.ToString()) Method. // One can sometimes observe nodes in the graph that are not present in any event of "<someLog>". // Further investigation shows that these nodes were however present in Logs previously // processed by the EventProcessor. Therefore it seems like the EventProcessor // is not working with a comepleteley fresh or cleared state when calling generateGraphFromString // multiple times. Even though one would not expect that from a static Method. // The problem seems to be solvable by creating a new instance of EventProcessor and (possibly?) // calling ep.freeRessources(). However this quickly leads to the next problem... // The ressources (memory) used by the EventProcessor are seemingly not garbage collected, // when the reference is nulled. This might be due to how Large Object Heap Collection works. // Processing multiple logs in a single EventProcessor instance // is not possible due to the "shared state" problem mentioned above. However, creating a new instance of // EventProcessor every time we need to process a log, is not a good solution either, because // memory from expired instances of EventProcessor is apparently not released. // Therefore one is forced to restart the program for each Log (frees memory) and also // choose a time step that leads to a number of partial Logs that fit into memory. Smaller time step // will result in more partial Logs and higher memory requirements (since we need to create // a new EventProcessor instance for each partial log.) Calling ep.freeRessources() for // each partial Log does not seem to do anything, neither does nulling the EventProcessor object.) // Debug: Output the Log for the current Interval // File.WriteAllText(Settings.outputDirectory + "\\" + outPath + "\\input-" + i + ".txt", splitMergedLogs[i].ToString()); /* ????????????????????????????????????????????????????????????????????????????? */ /* ????????????????????????????????????????????????????????????????????????????? */ /* When exactly are we supposed to call this ? */ ep = new EventProcessor(); ep.freeResources(); // generate QDFG for this time instant (step / snapshot) lastGraph = EventProcessor.generateGraphFromString(splitLogs[i].ToString()); DFTGraph workingGraph = new DFTGraph(lastGraph); // Debug Output graphical representation of the Graph for the current Interval //File.WriteAllText(Settings.outputDirectory + "\\" + outPath + "\\input-" + i + ".graphml",lastGraph.serializeGraphML()); // if stat collection is enabled, add stats for this time instant to the collection if (computeStats) { metrics.decorate(workingGraph); stats.addStats(workingGraph, i); foreach (DFTNode n in workingGraph.Vertices) { if (n.nameFTR.Contains("malware")) { logger.Debug("Node Features For " + n.nameFTR); foreach (DFTNodeFeature f in n.nodeFeatures) { logger.Debug(f.name); } logger.Debug("End of Node Features."); } } } verts = lastGraph.Vertices; foreach (DFTNode n in verts) { if (!processedIDs.ContainsKey(n.nameFTR)) { /* Somehow this information persists through separate method calls... * See advanced problem description above. * DFTNodeAttribute startTime = new DFTNodeAttribute(); * startTime.name = "start"; * startTime.value = i.ToString(); * n.nodeProperties.Add(startTime); */ // Workaround: Pass a dictionary containing the relevant time information to the GEXF Engine processedIDs.Add(n.nameFTR, i.ToString()); logger.Debug("ADDED NEW NODE: " + n.nameFTR + " -> " + i.ToString()); } } logger.Debug("IDs in Dict: " + processedIDs.Count); //splitMergedLogs[i].writeToFile(outPath + "\\SplitAndMergedLog-" + i + ".txt"); //String gSerialized = g.serializeGraphML(); //Utility.writeGraphToFile(gSerialized, outPath + "\\SplitAndMergedLog-" + i + ".graphml"); } } GEXFWriter.writeGraph(lastGraph, outPath + "\\dynamic.gexf", true, start, end, processedIDs); File.WriteAllText(Settings.outputDirectory + "\\" + outPath + "\\final" + ".graphml", lastGraph.serializeGraphML()); ep = new EventProcessor(); ep.freeResources(); logger.Warn("Wrote Dynamic Graph for: " + logPath); if (computeStats) { String statsFile = Settings.outputDirectory + "\\" + diOutput.Name + "\\" + "stats.txt"; stats.writeStatsToFile(statsFile); logger.Warn("Wrote Stats for: " + logPath + " to file: " + statsFile); } if (Settings.generateModel) { ModelBuilder.addModelData(stats, diOutput.Name); } } else { logger.Fatal(logPath + ": Not Enough Active Samples (" + numSamples + ", MIN: " + Settings.MIN_SAMPLES + ")"); } }
public static DFTGraph getReachableGraphFWD(DFTGraph g, DFTNode n) { return(g.getReachabilityGraphFWD(n)); }
public static DFTGraph getReachableGraphBWD(DFTGraph g, DFTNode n, DFTNodeType type) { return(g.getReachabilityGraphBWD(n, false, type)); }
/// <summary> /// Actual Implementation of the StaticLogProcessor. /// Generate Graph from Log File. Optionally generate Statistics and DataModel. /// </summary> /// <param name="logPath">Full Path to the Log File.</param> /// <param name="diOutput">DirectoryInfo of the output Directory.</param> /// <param name="computeStats">true if Statistics and Model should be generated; otherwise, false.</param> private void processLogStatic(String logPath, DirectoryInfo diOutput, bool computeStats) { Settings.MIN_SAMPLES = 1; // Static Logs only consist of 1 sample (the final state of the graph) Settings.timeStepMS = -1; // Time is not considered for static graphs. // Reset the external EventProcessor (LibDFT) EventProcessor ep = new EventProcessor(); ep.freeResources(); logger.Info("Processing Log (Static): " + logPath); // Init EventLog, Statistics & Metrics EventLog eventLog = new EventLog(logPath); QDFGStatCollection stats = new QDFGStatCollection(); CombinedMetrics metrics = new CombinedMetrics(); // Subpath for the Output Directory of this particular dynamic Log String outPath = diOutput.Name + "\\staticLog"; // Pre-process log file (Create Output Directories, Verify Log contains Nodes of interest) if (!preprocessLog(eventLog, logPath, outPath)) { return; } // Memory Check checkMemoryCount(logPath + ", Stage: init"); // Create Transformer for Event Log; Used to perform operations on EventLog EventLogTransformer transformer = new EventLogTransformer(eventLog); // Fix inconsistent time stamps & transform the absolute time representation into a relative one logger.Info("Fixing Time Info in Log: " + logPath); transformer.fixDates(); //eventLog.writeToFile(outPath + "\\sortedLog.txt"); // Memory Check checkMemoryCount(logPath + ", Stage: Fix Dates"); DFTGraph g = EventProcessor.generateGraphFromString(eventLog.ToString()); Utility.Graph.writeGraphToFile(g.serializeGraphML(), diOutput.Name + "\\StaticGraphG_" + Utility.IO.getFileNameFromPath(logPath) + ".graphml"); logger.Warn("Wrote Static Graph for: " + logPath); DFTGraph workingGraph = new DFTGraph(g); // if stat collection is enabled, add stats for this time instant to the collection if (computeStats) { logger.Info("Decorating graph: " + logPath); metrics.decorate(workingGraph); stats.addStats(workingGraph, 0); // logger.Info("Metrics for " + logPath); // printMetrics(workingGraph); } ep = new EventProcessor(); ep.freeResources(); if (computeStats) { String statsFile = Settings.outputDirectory + "\\" + diOutput.Name + "\\" + "stats.txt"; stats.writeStatsToFile(statsFile); logger.Warn("Wrote Stats for: " + logPath + " to file: " + statsFile); } if (Settings.generateModel) { ModelBuilder.addModelData(stats, diOutput.Name); } }