public void DiGraphTest()
{
DotGraph graph = new DotGraph("test", true);
var leaf1 = new DotNode("l1")
{
// Set all available properties
Shape = "doublecircle",
Label = "leaf1",
FontColor = "black",
Style = "",
Height = 0.5f
};
var leaf2 = new DotNode("l2")
{
// Set all available properties
Shape = "doublecircle",
Label = "leaf2",
FontColor = "red",
Style = "",
Height = 0.5f
};
var root = new DotNode("root")
{
// Set all available properties
Shape = "ellipse",
Label = "leaf1",
FontColor = "blue",
Style = "",
Height = 0.5f
};
graph.Add(leaf1);
graph.Add(leaf2);
graph.Add(root);
var edge1 = new DotArrow(root, leaf1)
{
ArrowHeadShape = "none"
};
var edge2 = new DotArrow(root, leaf2)
{
ArrowHeadShape = "normal"
};
graph.Add(edge1);
graph.Add(edge2);
string actual = graph.Compile().Replace("\r", "").Replace("\n", "");
string expected =
@"digraph test {l1 [ label=""leaf1"" shape=doublecircle fontcolor=black height=0.50]l2 [ label=""leaf2"" shape=doublecircle fontcolor=red height=0.50]root [ label=""leaf1"" shape=ellipse fontcolor=blue height=0.50]root->l1 [ arrowshape=none];root->l2 [ arrowshape=normal];}";
Assert.Equal(expected, actual);
}
private void MapRelationshipItemTypes(bool relsAsClasses)
{
Console.WriteLine("Mapping Relationship ItemTypes to graph...");
int relTypeCount = ArasExport.RelationshipTypes.getItemCount();
for (int i = 0; i < relTypeCount; i++)
{
var currentItemType = ArasExport.RelationshipTypes.getItemByIndex(i);
var propRels = currentItemType.getRelationships("Property");
(string sourceClassName, string targetClassName) = DetermineSourceAndTargetName(propRels);
if (sourceClassName == "" && targetClassName == "")
{
continue;
}
string customStyle;
string currentTypeName = currentItemType.getProperty("name", "");
if (targetClassName == "")
{
if (currentTypeName == "")
{
continue;
}
MapSpecialTypeAsClass(currentItemType);
targetClassName = currentTypeName;
customStyle = "dir = \"both\", arrowtail=\"odiamond\"";
}
else if (relsAsClasses)
{
MapRelsAsClasses(currentItemType, sourceClassName, targetClassName);
continue;
}
else
{
customStyle = $"label = \"{currentTypeName}\"";
}
var relationshipArrow = new DotArrow(sourceClassName, targetClassName, Graph)
{
CustomStyle = customStyle
};
Graph.GraphElements.Add(relationshipArrow);
}
Console.WriteLine("Relationship ItemTypes successfully mapped!");
}
private void MapRelsAsClasses(Item itemType, string sourceClassName, string targetClassName)
{
var relClass = MapSpecialTypeAsClass(itemType);
var sourceArrow = new DotArrow(relClass.Name, sourceClassName, Graph)
{
CustomStyle = "label = source_id"
};
Graph.GraphElements.Add(sourceArrow);
var targetArrow = new DotArrow(relClass.Name, targetClassName, Graph)
{
CustomStyle = "label = related_id"
};
Graph.GraphElements.Add(targetArrow);
}
private void MapPropertyRelations(Item itemType)
{
var itemProps = itemType.getItemsByXPath(".//Item[@type = 'Property' and data_type = 'item']");
int itemPropCount = itemProps.getItemCount();
for (int i = 0; i < itemPropCount; i++)
{
var currentProp = itemProps.getItemByIndex(i);
string dataSource = currentProp.getPropertyAttribute("data_source", "name", "");
if (dataSource == "")
{
continue;
}
var relationshipArrow =
new DotArrow(itemType.getProperty("name", $"\"{itemType.getID()}\""), dataSource, Graph)
{
CustomStyle = $"label = \"{currentProp.getProperty("name", "")}\""
};
Graph.GraphElements.Add(relationshipArrow);
}
}
private DotNode Visit(SyntaxNode <IN> node)
{
DotNode result = null;
var children = new List <DotNode>();
foreach (var n in node.Children)
{
var v = Visit(n);
children.Add(v);
}
if (node.IsByPassNode)
{
result = children[0];
}
else
{
result = Node(GetNodeLabel(node));
Graph.Add(result);
children.ForEach(c =>
{
if (c != null) // Prevent arrows with null destinations
{
var edge = new DotArrow(result, c)
{
// Set all available properties
ArrowHeadShape = "none"
};
Graph.Add(edge);
}
});
}
return(result);
}
private void MapPolyItemTypes()
{
Console.WriteLine("Mapping Poly-ItemTypes to graph...");
int polyTypeCount = ArasExport.PolyItemTypes.getItemCount();
for (int i = 0; i < polyTypeCount; i++)
{
var currentPolyType = ArasExport.PolyItemTypes.getItemByIndex(i);
var polyClass = MapSpecialTypeAsClass(currentPolyType);
var morphaeRels = currentPolyType.getRelationships("Morphae");
int morphaeCount = morphaeRels.getItemCount();
for (int j = 0; j < morphaeCount; j++)
{
var currentMorphae = morphaeRels.getItemByIndex(j);
var subClassItem = currentMorphae.getPropertyItem("related_id");
if (subClassItem == null)
{
continue;
}
var subClass =
Graph.GetDotClassByName(subClassItem.getProperty("name", "ClassHasNoName"), false);
subClass = subClass ?? MapSpecialTypeAsClass(subClassItem);
var relationshipArrow = new DotArrow(subClass, polyClass)
{
CustomStyle = "arrowhead = \"empty\""
};
Graph.GraphElements.Add(relationshipArrow);
}
}
Console.WriteLine("Poly-ItemTypes successfully mapped!");
}
private void processFunction(Function node, Function rootNode)
{
string id = node.Uid;
var currentVertex = createOutputNode(node);
foreach (var input in node.RootFunction.Inputs)
{
varStack.Push(input);
typeStack.Push(CntkType.Variable);
}
//add node's inputs
var inputs = node.Inputs;
for (int i = 0; i < inputs.Count; i++)
{
Variable input = inputs[i];
if (node.IsBlock && input.IsConstant)
{
continue;
}
DotNode vertex = null;
if (!input.IsOutput)
{
vertex = createNonOutputNode(input);
}
else
{
vertex = createOutputNode(input.Owner);
}
string label = string.IsNullOrEmpty(input.Name) ? input.Uid : input.Name;
label += "\n" + ShapeDescription(input);
DotArrow edge = new DotArrow(vertex, currentVertex);
edge.Label = label;
m_graph.Add(vertex);
m_graph.Add(currentVertex);
m_graph.Add(edge);
}
foreach (var output in node.Outputs)
{
//only final network outputs are drawn
if (node.Uid == rootNode.Uid)
{
var finalVertex = createVertex(output);
finalVertex.Label = output.Name + "\n" + ShapeDescription(output);
finalVertex.Shape = DotNodeShape.Egg;
finalVertex.FillColor = DotColor.Purple;
finalVertex.FixedSize = true;
finalVertex.Height = 1f;
finalVertex.Width = 1.3f;
finalVertex.PenWidth = 4;
//add nodes
m_graph.Add(currentVertex);
m_graph.Add(finalVertex);
}
}
//mark current node as visited
visitedNodes.Add(id);
}
private void processFunction(Function node, Function rootNode)
{
string id = node.Uid;
var currentVertex = LazyCreateNode(node);
foreach (var input in node.RootFunction.Inputs)
{
varStack.Push(input);
typeStack.Push(CntkType.Variable);
}
//add node's inputs
var inputs = node.Inputs;
for (int i = 0; i < inputs.Count; i++)
{
Variable input = inputs[i];
if (node.IsBlock && input.IsConstant)
{
continue;
}
DotNode vertex = null;
if (!input.IsOutput)
{
var name = input.Kind.ToString();
if (!string.IsNullOrEmpty(input.Name))
{
if (name.Equals("Parameter"))
{
name = input.Name;
}
else
{
name += "\n" + input.Name;
}
}
name += "\n" + ShapeDescription(input);
vertex = createVertex(input);
if (input.IsInput || input.IsPlaceholder)
{
vertex.Label = name;
//
vertex.FixedSize = true;
vertex.Height = 1f;
vertex.Width = 1.3f;
vertex.PenWidth = 4;
}
else if (string.IsNullOrEmpty(input.Name) && input.IsConstant && (input.Shape.Dimensions.Count == 0 || input.Shape.Dimensions[0] == 1))
{
string label1 = "";
var contView = new Constant(input).Value();
var value = new Value(contView);
switch (input.DataType)
{
case DataType.Float:
label1 = value.GetDenseData <float>(input)[0][0].ToString("N4", CultureInfo.InvariantCulture);
break;
case DataType.Double:
label1 = value.GetDenseData <double>(input)[0][0].ToString("N4", CultureInfo.InvariantCulture);
break;
case DataType.Float16:
label1 = (value.GetDenseData <float16>(input)[0][0]).ToString();
break;
default:
break;
}
vertex.Label = label1;
//
vertex.Height = 0.6f;
vertex.Width = 1f;
}
else
{
vertex.Label = name;
//
vertex.Height = 0.6f;
vertex.Width = 1f;
}
}
else
{
vertex = LazyCreateNode(input.Owner);
}
string label = string.IsNullOrEmpty(input.Name) ? input.Uid : input.Name;
label += "\n" + ShapeDescription(input);
DotArrow edge = new DotArrow(vertex, currentVertex);
edge.Label = label;
m_graph.Add(vertex);
m_graph.Add(currentVertex);
m_graph.Add(edge);
}
foreach (var output in node.Outputs)
{
//only final network outputs are drawn
if (node.Uid == rootNode.Uid)
{
var finalVertex = createVertex(output);
finalVertex.Label = output.Name + "\n" + ShapeDescription(output);
finalVertex.Shape = DotNodeShape.Egg;
finalVertex.FillColor = DotColor.Purple;
finalVertex.FixedSize = true;
finalVertex.Height = 1f;
finalVertex.Width = 1.3f;
finalVertex.PenWidth = 4;
//add nodes
m_graph.Add(currentVertex);
m_graph.Add(finalVertex);
}
}
//mark current node as visited
visitedNodes.Add(id);
}
static void Main(string[] args)
{
/*
* This program gets 2 inputs:
* List of all functions in cleaned format from IDA Pro as a list:
* <function_name> <address> <length>
*
* And trace block information in cleaned format from DynamoRIO block tracking tool "drcov":
* <block_adress>
*
* No other input is neccesary.
*
* Program outputs a graph in .dot format to be used with any GraphViz visualizer.
*
* There is sample input and output included in this repository.
*
* All numbers are in base 16. Detailed information about this algorithm can be found in the masters thesis:
* MACHINE CODE INSTRUMENTATION FOR BLOCK RUNTIME STATISTICAL ANALYSIS AND PREDICTION
*/
// Input
const string FUNCTION_LIST_INPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\fun_list_all.txt";
const string TRACED_BLOCKS_INPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\doc_cleaned.log";
// Input seperator for function data columns <function_name>_SEPERATOR_<address>_SEPERATOR_<length>
const string FUNCTION_SEPERATOR = "_SEPERATOR_";
// Output
const string NODE_STATS_OUTPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\node_stats.txt";
const string GRAPH_OUTPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\graph.dot";
const string TABLE_OUTPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\node_table.txt";
const string NODE_LIST_OUTPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\node_sequence.txt";
// We want to create a list of all functions with their beginning and end adresses
List <string> functionsAsText = File.ReadAllText(FUNCTION_LIST_INPUT_PATH)
.Split('\n')
.Select(l => l.Trim())
.ToList();
var functionTextPattern = new Regex($"([^\\s]+){FUNCTION_SEPERATOR}([^\\s]+){FUNCTION_SEPERATOR}([^\\s]+)");
List <MachineCodeFunction> functions = new List <MachineCodeFunction>();
foreach (string functionTextLine in functionsAsText)
{
var match = functionTextPattern.Match(functionTextLine);
functions.Add(new MachineCodeFunction()
{
Name = match.Groups[1].Value,
Start = Convert.ToInt64(match.Groups[2].Value, 16),
End = Convert.ToInt64(match.Groups[2].Value, 16) + Convert.ToInt64(match.Groups[3].Value, 16),
});
}
// We know have a block trace, for each block we figure out which function that is and replace it with a function
List <string> linesBlocks = File.ReadAllText(TRACED_BLOCKS_INPUT_PATH)
.Split('\n', StringSplitOptions.RemoveEmptyEntries)
.Select(l => l.Trim())
.ToList();
List <MachineCodeFunction> functionSequence = new List <MachineCodeFunction>();
foreach (string block in linesBlocks)
{
long address = Convert.ToInt64(block, 16);
foreach (MachineCodeFunction idaFun in functions)
{
if (address >= idaFun.Start && address <= idaFun.End)
{
functionSequence.Add(idaFun);
break;
}
}
}
// Add start and end
functionSequence = functionSequence.Prepend(new MachineCodeFunction(customShortName: "Start")
{
Name = "Start",
}).ToList();
functionSequence = functionSequence.Append(new MachineCodeFunction(customShortName: "Exit")
{
Name = "Exit",
}).ToList();
// Now we reduce the function trace list by removing repeating blocks within the same func (eg. A A A B B C A A -> A B C A)
List <MachineCodeFunction> functionListBlocksJoined = new List <MachineCodeFunction>();
foreach (var function in functionSequence)
{
if (functionListBlocksJoined.Count == 0 || functionListBlocksJoined.Last() != function)
{
functionListBlocksJoined.Add(function);
}
}
string outputNodeSeq = string.Empty;
foreach (var function in functionListBlocksJoined)
{
outputNodeSeq += function.ShortName + Environment.NewLine;
}
File.WriteAllText(NODE_LIST_OUTPUT_PATH, outputNodeSeq);
string nodeStats = string.Empty;
// We also calculate how often each function is called and store that as extra info
functionListBlocksJoined.GroupBy(fun => fun)
.ToList()
.ForEach(functionCalls =>
{
var info = functionCalls.First();
info.Calls = functionCalls.Count();
nodeStats += $"Node nr. {info.ShortName} ({info.Name}), called count: {info.Calls}\r\n";
});
// Then calculate avarage and stdev, if it's more than 1 stdev color change, 2 stdev extra color change
var uniqueFunctions = functionListBlocksJoined.Distinct().ToList();
(double stdev, double avarage) = GetStandardDeviation(uniqueFunctions.Select(node => (double)node.Calls).ToList());
nodeStats = $"Avarage calls: {avarage}, standard deviation: {stdev}\r\n" + nodeStats;
foreach (var function in uniqueFunctions)
{
if (function.Calls > avarage + 2 * stdev)
{
function.NodeColor = DotColor.Green4;
nodeStats += $"Node nr. {function.ShortName} ({function.Name}), is called 2 STDEV more than AVG.\r\n";
}
else if (function.Calls > avarage + 1 * stdev)
{
function.NodeColor = DotColor.Olivedrab;
nodeStats += $"Node nr. {function.ShortName} ({function.Name}), is called 1 STDEV more than AVG.\r\n";
}
else if (function.Calls < avarage - 2 * stdev)
{
function.NodeColor = DotColor.Red1;
nodeStats += $"Node nr. {function.ShortName} ({function.Name}), is called 2 STDEV less than AVG.\r\n";
}
else if (function.Calls < avarage - 1 * stdev)
{
function.NodeColor = DotColor.Red4;
nodeStats += $"Node nr. {function.ShortName} ({function.Name}), is called 1 STDEV less than AVG.\r\n";
}
}
File.WriteAllText(NODE_STATS_OUTPUT_PATH, nodeStats);
// For each node calculate all its successors for table visualization
for (int i = 0; i < functionListBlocksJoined.Count; i++)
{
var @this = functionListBlocksJoined[i];
if (i + 1 != functionListBlocksJoined.Count)
{
@this.Next.Add(functionListBlocksJoined[i + 1]);
}
}
string outputTable = string.Empty;
var uniqueFunctionsNoRepeatingBlocks = functionListBlocksJoined.Distinct();
List <(MachineCodeFunction, List <MachineCodeFunctionGrouping>)> graphAsTable = new List <(MachineCodeFunction, List <MachineCodeFunctionGrouping>)> ();
foreach (var function in uniqueFunctionsNoRepeatingBlocks)
{
List <MachineCodeFunctionGrouping> tableEntry = function.Next.GroupBy(
functionNext => functionNext,
functionNext => functionNext,
(key, grouping) => new MachineCodeFunctionGrouping
{
ShortName = key.ShortName,
PreviousElement = function.ShortName,
NodeColor = function.NodeColor,
Key = key,
Probability = grouping.Count() / (double)function.Next.Count
}).ToList();
outputTable += function.ShortName + Environment.NewLine;
foreach (var group in tableEntry)
{
outputTable += group.ShortName + $" {Math.Round(group.Probability, 2)} ";
}
outputTable += Environment.NewLine;
graphAsTable.Add((function, tableEntry));
}
File.WriteAllText(TABLE_OUTPUT_PATH, outputTable);
// GraphViz export
var directedGraph = new DotGraph("BlockTraceGraph", true);
foreach (var(node, _) in graphAsTable)
{
var graphNode = new DotNode(node.ShortName)
{
Shape = DotNodeShape.Ellipse,
Label = node.ShortName + $"({node.Calls})",
// FillColor = node.NodeColor != null ? DotColor.Grey100 : DotColor.White,
FontColor = node.NodeColor ?? DotColor.Black,
Style = (node.NodeColor != null ? DotNodeStyle.Bold : DotNodeStyle.Default),
Height = 0.5f,
};
directedGraph.Add(graphNode);
}
foreach (var(_, edges) in graphAsTable)
{
// Let's do some coloring
// If all edges have the same weights color blue
// If there is one and ONLY one that is higher prob than everyone then GREEN
// if there is one and ONLY one that is higher prob than everyone then RED
// If only GREY
bool areAllSameProbability = edges.All(e => e.Probability == edges[0].Probability);
if (!areAllSameProbability)
{
var maxProbability = edges.Max(e => e.Probability);
var isOnly = edges.Count(e => e.Probability == maxProbability) == 1;
if (isOnly)
{
edges.First(e => e.Probability == maxProbability).Color = DotColor.Green3;
}
var minProbability = edges.Min(e => e.Probability);
var isOnlyMin = edges.Count(e => e.Probability == minProbability) == 1;
if (isOnlyMin)
{
edges.First(e => e.Probability == minProbability).Color = DotColor.Red1;
}
}
foreach (var edge in edges)
{
var arrow = new DotArrow(edge.PreviousElement, edge.ShortName)
{
ArrowLabel = Math.Round(edge.Probability, 2).ToString()
};
if (areAllSameProbability)
{
arrow.ArrowColor = DotColor.Blue;
}
if (edge.Color != null)
{
arrow.ArrowColor = edge.Color.Value;
}
if (edges.Count == 1)
{
arrow.ArrowColor = DotColor.Gray;
}
directedGraph.Add(arrow);
}
}
// Indented version
var dot = directedGraph.Compile(false);
// Save it to a file
File.WriteAllText(GRAPH_OUTPUT_PATH, dot);
}
