public void Considers_Subgraph_equivalent_even_when_elements_are_different(
string graphId,
DotNode dotNode1,
DotNode dotNode2)
{
var graph1 = new SubGraph(graphId);
graph1.Elements.Add(dotNode1);
var graph2 = new SubGraph(graphId);
graph2.Elements.Add(dotNode2);
using var a = new AssertionScope();
graph1.Should().BeEquivalentTo(graph2);
var cluster = graph1.Elements.First() as DotNode;
var cluster2 = graph2.Elements.First() as DotNode;
cluster.Should().NotBeEquivalentTo(cluster2);
}
public void EdgeWithNodeToNode()
{
var helloNode = new DotNode("hello");
var worldNode = new DotNode("world");
var graph = new DotGraph("TestGraph")
{
Elements =
{
helloNode,
worldNode,
new DotEdge(helloNode, worldNode)
}
};
var compiled = graph.Compile();
Check.That(compiled).HasSameValueAs("graph TestGraph { hello ; world ; hello -- world; }");
}
public void Visit(DotNode node)
{
var self = node;
var theMostInner = self;
while (!(self is null))
{
theMostInner = self;
self = self.Root as DotNode;
}
var ident = (IdentifierNode)theMostInner.Root;
if (node == theMostInner && Scope.ScopeSymbolTable.SymbolIsOfType <TableSymbol>(ident.Name))
{
if (theMostInner.Expression is DotNode dotNode)
{
var col = (IdentifierNode)dotNode.Root;
Visit(new AccessColumnNode(col.Name, ident.Name, TextSpan.Empty));
}
else
{
var col = (IdentifierNode)theMostInner.Expression;
Visit(new AccessColumnNode(col.Name, ident.Name, TextSpan.Empty));
}
return;
}
self = node;
while (!(self is null))
{
self.Root.Accept(this);
self.Expression.Accept(this);
self.Accept(_visitor);
self = self.Expression as DotNode;
}
}
public virtual void Visit(DotNode node)
{
var exp = Nodes.Pop();
var root = Nodes.Pop();
if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(node.ReturnType))
{
Nodes.Push(new DotNode(root, exp, node.IsOuter, node.Name, typeof(IDynamicMetaObjectProvider)));
}
else
{
if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(root.ReturnType))
{
Nodes.Push(new DotNode(root, exp, node.IsOuter, node.Name, typeof(IDynamicMetaObjectProvider)));
}
else
{
Nodes.Push(new DotNode(root, exp, node.IsOuter, node.Name, exp.ReturnType));
}
}
}
public void Visit(IASTNode node)
{
// todo : currently expects that the individual with expressions apply to the same sql table join.
// This may not be the case for joined-subclass where the property values
// might be coming from different tables in the joined hierarchy. At some
// point we should expand this to support that capability. However, that has
// some difficulties:
// 1) the biggest is how to handle ORs when the individual comparisons are
// linked to different sql joins.
// 2) here we would need to track each comparison individually, along with
// the join alias to which it applies and then pass that information
// back to the FromElement so it can pass it along to the JoinSequence
if (node is DotNode)
{
DotNode dotNode = ( DotNode )node;
FromElement fromElement = dotNode.FromElement;
if (_referencedFromElement == null)
{
_referencedFromElement = fromElement;
_joinAlias = ExtractAppliedAlias(dotNode);
// todo : temporary
// needed because currently persister is the one that
// creates and renders the join fragments for inheritence
// hierarchies...
if (_joinAlias != _referencedFromElement.TableAlias)
{
throw new InvalidWithClauseException("with clause can only reference columns in the driving table");
}
}
}
else if (node is ParameterNode)
{
ApplyParameterSpecification(((ParameterNode)node).HqlParameterSpecification);
}
else if (node is IParameterContainer)
{
ApplyParameterSpecifications(( IParameterContainer )node);
}
}
private void AddModeledImages(DotGraph graph, PlatformInfo[] platforms)
{
IEnumerable <IEnumerable <string> > images = platforms.Select(platform => platform.Tags.Select(tag => tag.FullyQualifiedName));
LoadImageManifests(images);
foreach (PlatformInfo platform in platforms)
{
IEnumerable <string> tags = platform.Tags.Select(tag => tag.FullyQualifiedName);
DotNode imageNode = AddImageNode(graph, tags, Color.Navy, platform.FinalStageFromImage);
if (platform.FinalStageFromImage is not null)
{
var myEdge = new DotEdge(imageNode.Id, _nodeCache[platform.FinalStageFromImage].Id);
myEdge.Attributes.Head.Arrowhead = DotArrowheadShape.Normal;
myEdge.Attributes.Tail.Arrowhead = DotArrowheadShape.None;
myEdge.Attributes.Color = Color.Black;
myEdge.Attributes.Style.LineStyle = DotLineStyle.Dashed;
graph.Edges.Add(myEdge);
}
}
}
private DotNode Leaf(IN type, string value)
{
string label = type.ToString();
label += "\n";
var esc = value.Replace("\"", "\\\"");
label += "\\\"" + esc + "\\\"";
var node = new DotNode(NodeCounter.ToString())
{
// Set all available properties
Shape = "doublecircle",
Label = label,
FontColor = "",
Style = "",
Height = 0.5f
};
NodeCounter++;
Graph.Add(node);
return(node);
}
public override object GetResult()
{
var graph = new DotGraph(directed: true);
graph.Attributes.Label = "Abstract Syntax Tree";
graph.Attributes.EdgeShape = DotEdgeShape.Orthogonal;
graph.Nodes.AddRange(_nodes.Select(vnode =>
{
var dotNode = new DotNode(vnode.Id);
dotNode.Attributes.Label = "SyntaxTreeNode[" + vnode.Node.RuleName + "]{ " + (vnode.Node.ParsedText is null || vnode.Node.ParsedText == string.Empty ? "[[NULL]]" : vnode.Node.ParsedText) + " }{ " + (vnode.Node.Rest is null || vnode.Node.Rest == string.Empty ? "[[NULL]]" : vnode.Node.Rest) + " }";
dotNode.Attributes.Shape = DotNodeShape.Box;
dotNode.Attributes.Style.FillStyle = DotNodeFillStyle.Normal;
return(dotNode);
}));
graph.Edges.AddRange(_links.Select(link => {
var edge = new DotEdge(link.SourceNode.Id, link.DestinationNode.Id);
edge.Attributes.Label = link.Label ?? "";
return(edge);
}));
return(graph.Build());
}
private DotNode Visit(SyntaxNode <IN> node)
{
DotNode result = null;
var children = new List <DotNode>();
foreach (var n in node.Children)
{
var v = Visit(n);
children.Add(v);
}
if (node.IsByPassNode)
{
result = children[0];
}
else
{
result = Node(GetNodeLabel(node));
Graph.Add(result);
children.ForEach(c =>
{
if (c != null) // Prevent arrows with null destinations
{
var edge = new DotArrow(result, c)
{
// Set all available properties
ArrowHeadShape = "none"
};
Graph.Add(edge);
}
});
}
return(result);
}
private void InitialiseGraph()
{
Graph = new DotGraph {
IsDiGraph = true
};
Graph.AddAttribute("fontname", "Calibri");
Graph.AddAttribute("fontsize", "10");
var baseNode = new DotNode();
baseNode.AddAttribute("shape", "record");
baseNode.AddAttribute("fontname", "Calibri");
baseNode.AddAttribute("fontsize", "10");
Graph.GraphElements.Add(baseNode);
var baseEdge = new DotEdge();
baseEdge.AddAttribute("shape", "record");
baseEdge.AddAttribute("fontname", "Calibri");
baseEdge.AddAttribute("fontsize", "10");
baseEdge.AddAttribute("arrowhead", "normal");
Graph.GraphElements.Add(baseEdge);
}
DotNode LazyCreateNode(Function node)
{
var primitiveOperationsMap = PrimitiveOperationsMap();
DotNode vertex = null;
if (node.IsPrimitive && !node.IsBlock && node.Outputs.Count == 1 && node.Output.Name == node.Name)
{
//
var operationName = primitiveOperationsMap.ContainsKey(node.OpName)? primitiveOperationsMap[node.OpName]: node.OpName;
bool renderAsPrimitive = operationName.Count() <= 4;
var size = renderAsPrimitive ? 0.4f : 0.6f;
vertex = createVertex(node);
vertex.Label = operationName;
//
vertex.FixedSize = renderAsPrimitive ? true : false;
vertex.Height = size;
vertex.Width = size;
vertex.FontSize = renderAsPrimitive && operationName.Length == 1 ? 20 : 12;
vertex.PenWidth = node.OpName != "Pass" && node.OpName != "ParameterOrder" ? 4 : 1;
}
else
{
string functionName = string.IsNullOrEmpty(node.Name) ? "" : "\n" + node.Name + "()";
vertex = createVertex(node);
//
vertex.Label = node.OpName + functionName;
vertex.FixedSize = true;
vertex.Height = 1;
vertex.Width = 1.3f;
vertex.PenWidth = node.OpName != "Pass" && node.OpName != "ParameterOrder" ? 4 : 1;
}
return(vertex);
}
public override void Apply(CstDsl cstDsl)
{
var node = new DotNode(cstDsl.IdShort);
node.Attributes.Label = "DSL";
_graph.Nodes.Add(node);
var declarationAreaNode = CreateAttributeNode(nameof(cstDsl.Declarations));
_graph.Nodes.Add(declarationAreaNode);
foreach (var declaration in cstDsl.Declarations)
{
declaration.Visit(this);
_graph.Edges.Add(declarationAreaNode.Id, declaration.IdShort);
}
_graph.Edges.Add(node.Id, declarationAreaNode.Id);
if (!cstDsl.CodeEntities.Any())
{
return;
}
var assetsAreaNode = CreateAttributeNode(nameof(cstDsl.CodeEntities));
_graph.Nodes.Add(assetsAreaNode);
foreach (var asset in cstDsl.CodeEntities)
{
asset.Visit(this);
_graph.Edges.Add(assetsAreaNode.Id, asset.IdShort);
}
_graph.Edges.Add(node.Id, assetsAreaNode.Id);
}
private DotNode createVertex(Variable node)
{
var n = new DotNode(node.Uid);
if (node.IsInput)
{
n.Style = DotNodeStyle.Filled;
n.Shape = DotNodeShape.Egg;
n.FillColor = DotColor.Forestgreen;
}
else if (node.IsPlaceholder)
{
n.Style = DotNodeStyle.Filled;
n.Shape = DotNodeShape.Invhouse;
n.FillColor = DotColor.Grey;
}
else if (node.IsParameter)
{
n.Style = DotNodeStyle.Filled;
n.Shape = DotNodeShape.Note;
n.FillColor = DotColor.Palegreen;
}
else if (node.IsConstant)
{
n.Style = DotNodeStyle.Filled;
n.Shape = DotNodeShape.Rect;
n.FillColor = DotColor.Grey;
}
else
{
n.Style = DotNodeStyle.Filled;
n.Shape = DotNodeShape.Rectangle;
n.FillColor = DotColor.Lemonchiffon1;
}
return(n);
}
public void VisitLeave(DotNode node)
{
tabs--;
}
private void SetConstantValue(DotNode node, string text, object value)
{
if (log.IsDebugEnabled())
{
log.Debug("setConstantValue() {0} -> {1} {2}", text, value, value.GetType().Name);
}
node.ClearChildren(); // Chop off the rest of the tree.
if (value is string)
{
node.Type = HqlSqlWalker.QUOTED_String;
}
else if (value is char)
{
node.Type = HqlSqlWalker.QUOTED_String;
}
else if (value is byte)
{
node.Type = HqlSqlWalker.NUM_INT;
}
else if (value is short)
{
node.Type = HqlSqlWalker.NUM_INT;
}
else if (value is int)
{
node.Type = HqlSqlWalker.NUM_INT;
}
else if (value is long)
{
node.Type = HqlSqlWalker.NUM_LONG;
}
else if (value is double)
{
node.Type = HqlSqlWalker.NUM_DOUBLE;
}
else if (value is decimal)
{
node.Type = HqlSqlWalker.NUM_DECIMAL;
}
else if (value is float)
{
node.Type = HqlSqlWalker.NUM_FLOAT;
}
else
{
node.Type = HqlSqlWalker.CONSTANT;
}
IType type;
try
{
type = TypeFactory.HeuristicType(value.GetType().Name);
}
catch (MappingException me)
{
throw new QueryException(me);
}
if (type == null)
{
throw new QueryException(LiteralProcessor.ErrorCannotDetermineType + node.Text);
}
try
{
ILiteralType literalType = (ILiteralType)type;
NHibernate.Dialect.Dialect dialect = _walker.SessionFactoryHelper.Factory.Dialect;
node.Text = literalType.ObjectToSQLString(value, dialect);
}
catch (Exception e)
{
throw new QueryException(LiteralProcessor.ErrorCannotFormatLiteral + node.Text, e);
}
node.DataType = type;
node.SetResolvedConstant(text);
}
public virtual void Visit(DotNode node)
{
node.Root.Accept(this);
node.Expression.Accept(this);
node.Accept(Visitor);
}
public DotArrow(DotNode startNode, DotNode targetNode) : this(startNode.Name, targetNode.Name)
{
}
void CreateFromJoinElement(
IASTNode path,
IASTNode alias,
int joinType,
IASTNode fetchNode,
IASTNode propertyFetch,
IASTNode with)
{
bool fetch = fetchNode != null;
if (fetch && IsSubQuery)
{
throw new QueryException("fetch not allowed in subquery from-elements");
}
// the incoming "path" can be either:
// 1) an implicit join path (join p.address.city)
// 2) an entity-join (join com.acme.User)
//
// so make the proper interpretation here...
var entityJoinReferencedPersister = ResolveEntityJoinReferencedPersister(path);
if (entityJoinReferencedPersister != null)
{
var entityJoin = CreateEntityJoin(entityJoinReferencedPersister, alias, joinType, with);
((FromReferenceNode)path).FromElement = entityJoin;
SetPropertyFetch(entityJoin, propertyFetch, alias);
return;
}
// The path AST should be a DotNode, and it should have been evaluated already.
if (path.Type != DOT)
{
throw new SemanticException("Path expected for join!");
}
DotNode dot = ( DotNode )path;
//JoinType hibernateJoinType = JoinProcessor.ToHibernateJoinType( joinType );
JoinType hibernateJoinType = _impliedJoinType;
dot.JoinType = hibernateJoinType; // Tell the dot node about the join type.
dot.Fetch = fetch;
// Generate an explicit join for the root dot node. The implied joins will be collected and passed up
// to the root dot node.
dot.Resolve(true, false, alias == null ? null : alias.Text);
FromElement fromElement;
if (dot.DataType != null && dot.DataType.IsComponentType)
{
var factory = new FromElementFactory(CurrentFromClause, dot.GetLhs().FromElement, dot.PropertyPath, alias == null ? null : alias.Text, null, false);
fromElement = factory.CreateComponentJoin((ComponentType)dot.DataType);
}
else
{
fromElement = dot.GetImpliedJoin();
if (fromElement == null)
{
throw new InvalidPathException("Invalid join: " + dot.Path);
}
SetPropertyFetch(fromElement, propertyFetch, alias);
if (with != null)
{
if (fetch)
{
throw new SemanticException("with-clause not allowed on fetched associations; use filters");
}
HandleWithFragment(fromElement, with);
}
if (fromElement.Parent == null)
{
fromElement.FromClause.AddChild(fromElement);
}
}
if (log.IsDebugEnabled())
{
log.Debug("createFromJoinElement() : {0}", _printer.ShowAsString(fromElement, "-- join tree --"));
}
}
public void Visit(DotNode node)
{
}
public void ModifyNodeIdentifierWithNullIdentifierThrowsException()
{
var node = new DotNode("test");
Check.ThatCode(() => node.Identifier = null).Throws <ArgumentException>();
}
public static void DebugExpressionGraph(VFXGraph graph, VFXExpressionContextOption option, string fileName = "expGraph.dot")
{
var expressionGraph = new VFXExpressionGraph();
expressionGraph.CompileExpressions(graph, option, true);
var mainExpressions = new Dictionary <VFXExpression, List <string> >();
FillMainExpressions(mainExpressions, expressionGraph.GPUExpressionsToReduced, expressionGraph);
FillMainExpressions(mainExpressions, expressionGraph.CPUExpressionsToReduced, expressionGraph);
var expressions = expressionGraph.Expressions;
DotGraph dotGraph = new DotGraph();
var expressionsToDot = new Dictionary <VFXExpression, DotNode>();
foreach (var exp in expressions)
{
var dotNode = new DotNode();
string name = exp.GetType().Name;
name += " " + exp.valueType.ToString();
string valueStr = GetExpressionValue(exp);
if (!string.IsNullOrEmpty(valueStr))
{
name += string.Format(" ({0})", valueStr);
}
dotNode.attributes[DotAttribute.Shape] = DotShape.Box;
if (mainExpressions.ContainsKey(exp))
{
string allOwnersStr = string.Empty;
foreach (var str in mainExpressions[exp])
{
allOwnersStr += "\n" + str;
}
name += string.Format("{0}", allOwnersStr);
if (exp.IsAny(VFXExpression.Flags.NotCompilableOnCPU))
{
dotNode.attributes[DotAttribute.Color] = DotColor.Orange;
}
else if (exp.Is(VFXExpression.Flags.Constant))
{
dotNode.attributes[DotAttribute.Color] = DotColor.SteelBlue;
}
else
{
dotNode.attributes[DotAttribute.Color] = DotColor.Cyan;
}
}
else if (exp.IsAny(VFXExpression.Flags.NotCompilableOnCPU))
{
dotNode.attributes[DotAttribute.Color] = DotColor.Yellow;
}
else if (exp.Is(VFXExpression.Flags.Constant))
{
dotNode.attributes[DotAttribute.Color] = DotColor.SlateGray;
}
else if (exp.Is(VFXExpression.Flags.Foldable))
{
dotNode.attributes[DotAttribute.Color] = DotColor.LightGray;
}
if (dotNode.attributes.ContainsKey(DotAttribute.Color))
{
dotNode.attributes[DotAttribute.Style] = DotStyle.Filled;
}
dotNode.Label = name;
expressionsToDot[exp] = dotNode;
dotGraph.AddElement(dotNode);
}
foreach (var exp in expressionsToDot)
{
var parents = exp.Key.parents;
for (int i = 0; i < parents.Length; ++i)
{
var dotEdge = new DotEdge(expressionsToDot[parents[i]], exp.Value);
if (parents.Length > 1)
{
dotEdge.attributes[DotAttribute.HeadLabel] = i.ToString();
}
dotGraph.AddElement(dotEdge);
}
}
var basePath = Application.dataPath;
basePath = basePath.Replace("/Assets", "");
basePath = basePath.Replace("/", "\\");
var outputfile = basePath + "\\GraphViz\\output\\" + fileName;
dotGraph.OutputToDotFile(outputfile);
var proc = new Process();
proc.StartInfo.WindowStyle = ProcessWindowStyle.Hidden;
proc.StartInfo.CreateNoWindow = true;
proc.StartInfo.UseShellExecute = false;
proc.StartInfo.FileName = "C:\\Windows\\system32\\cmd.exe";
var path = basePath + "\\GraphViz\\Postbuild.bat";
proc.StartInfo.Arguments = "/c" + path + " \"" + outputfile + "\"";
proc.EnableRaisingEvents = true;
proc.Start();
}
static int count = 0; //1
public void createDOTfile(Node root) //Args: root of the tree, string to specify where the generated DOT file will be saved.
{
if (count == 0)
{
root.unique_id = "-1";
} //Set the first iteration (root) ID to 0
if (root.isRoot) //Create root node on first iteration
{
tree = new DotGraph("Tree", true); //second parameter should always be true as this refers to a directed graph, which a decision tree always is
string x;
if (root.attribute.DataType != typeof(string))
{
x = "att: " + root.attribute + " thresh: " + root.threshold;
}
else
{
x = root.attribute.ColumnName + ": binary split";
}
var root_node = new DotNode(root.unique_id)
{
Shape = DotNodeShape.Ellipse,
Label = x,
FillColor = Color.Coral,
FontColor = Color.Black,
Style = DotNodeStyle.Filled,
Width = 0.5f,
Height = 0.5f
};
tree.Elements.Add(root_node);
}
if (root.children != null && root.children.Nodes.Count != 0)
{
for (int x = 0; x < root.children.Nodes.Count; x++)
{
root.children.Nodes[x].unique_id = count.ToString();
count++; //4
if (root.children.Nodes[x].attribute != null) //if not a leaf node
{
string c = root.children.Nodes[x].attribute.ColumnName + ": binary split";
if ((root.children.Nodes[x].attribute.DataType != typeof(string)))
{
string a = root.children.Nodes[x].attribute.ColumnName;
string b = root.children.Nodes[x].threshold.ToString();
c = "att: " + a + " " + "thresh: " + b;
}
var this_node = new DotNode(root.children.Nodes[x].unique_id)
{
Shape = DotNodeShape.Ellipse,
Label = c,
FillColor = Color.Coral,
FontColor = Color.Black,
Style = DotNodeStyle.Bold,
Width = 0.5f,
Height = 0.5f
};
tree.Elements.Add(this_node);
string parent_id = root.children.Nodes[x].parent.unique_id;
var this_edge = new DotEdge(parent_id, root.children.Nodes[x].unique_id); // Create and add an edge with identifiers //Does this create two blank nodes as we are calling by string?
tree.Elements.Add(this_edge);
createDOTfile(root.children.Nodes[x]);
}
else //attribute is null, thus is a leaf node
{
if (root.children.Nodes[x].label != "x") //Ensure mutated nodes with no attribute nor label are added to the tree //PATCH
{
var this_node = new DotNode(root.children.Nodes[x].unique_id)
{
Shape = DotNodeShape.Ellipse,
Label = root.children.Nodes[x].label,
FillColor = Color.LightGray,
FontColor = Color.Black,
Style = DotNodeStyle.Filled,
Width = 0.5f,
Height = 0.5f
};
tree.Elements.Add(this_node);
string parent_id = root.children.Nodes[x].parent.unique_id;
var this_edge = new DotEdge(parent_id, root.children.Nodes[x].unique_id); // Create and add an edge with identifiers //Does this create two blank nodes as we are calling by string?
tree.Elements.Add(this_edge);
}
}
}
}
if (root.isRoot) //Ensure the tree is created only once recursion returns to the root
{
Console.WriteLine("Nodes in DotGraph: " + tree.Elements.Count); //Testing
int edges = 0;
int nodes = 0;
foreach (var x in tree.Elements)
{
if (x.GetType() == typeof(DotEdge))
{
edges += 1;
}
if (x.GetType() == typeof(DotNode))
{
nodes += 1;
}
}
Console.WriteLine("Number of nodes in tree: " + nodes);
Console.WriteLine("Number of edges in tree: " + edges);
var dot = tree.Compile();
// Save it to a file
File.WriteAllText("myFile.dot", dot);
}
}
static void Main(string[] args)
{
/*
* This program gets 2 inputs:
* List of all functions in cleaned format from IDA Pro as a list:
* <function_name> <address> <length>
*
* And trace block information in cleaned format from DynamoRIO block tracking tool "drcov":
* <block_adress>
*
* No other input is neccesary.
*
* Program outputs a graph in .dot format to be used with any GraphViz visualizer.
*
* There is sample input and output included in this repository.
*
* All numbers are in base 16. Detailed information about this algorithm can be found in the masters thesis:
* MACHINE CODE INSTRUMENTATION FOR BLOCK RUNTIME STATISTICAL ANALYSIS AND PREDICTION
*/
// Input
const string FUNCTION_LIST_INPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\fun_list_all.txt";
const string TRACED_BLOCKS_INPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\doc_cleaned.log";
// Input seperator for function data columns <function_name>_SEPERATOR_<address>_SEPERATOR_<length>
const string FUNCTION_SEPERATOR = "_SEPERATOR_";
// Output
const string NODE_STATS_OUTPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\node_stats.txt";
const string GRAPH_OUTPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\graph.dot";
const string TABLE_OUTPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\node_table.txt";
const string NODE_LIST_OUTPUT_PATH = @"C:\Users\edza\Desktop\mag\case_study_big\node_sequence.txt";
// We want to create a list of all functions with their beginning and end adresses
List <string> functionsAsText = File.ReadAllText(FUNCTION_LIST_INPUT_PATH)
.Split('\n')
.Select(l => l.Trim())
.ToList();
var functionTextPattern = new Regex($"([^\\s]+){FUNCTION_SEPERATOR}([^\\s]+){FUNCTION_SEPERATOR}([^\\s]+)");
List <MachineCodeFunction> functions = new List <MachineCodeFunction>();
foreach (string functionTextLine in functionsAsText)
{
var match = functionTextPattern.Match(functionTextLine);
functions.Add(new MachineCodeFunction()
{
Name = match.Groups[1].Value,
Start = Convert.ToInt64(match.Groups[2].Value, 16),
End = Convert.ToInt64(match.Groups[2].Value, 16) + Convert.ToInt64(match.Groups[3].Value, 16),
});
}
// We know have a block trace, for each block we figure out which function that is and replace it with a function
List <string> linesBlocks = File.ReadAllText(TRACED_BLOCKS_INPUT_PATH)
.Split('\n', StringSplitOptions.RemoveEmptyEntries)
.Select(l => l.Trim())
.ToList();
List <MachineCodeFunction> functionSequence = new List <MachineCodeFunction>();
foreach (string block in linesBlocks)
{
long address = Convert.ToInt64(block, 16);
foreach (MachineCodeFunction idaFun in functions)
{
if (address >= idaFun.Start && address <= idaFun.End)
{
functionSequence.Add(idaFun);
break;
}
}
}
// Add start and end
functionSequence = functionSequence.Prepend(new MachineCodeFunction(customShortName: "Start")
{
Name = "Start",
}).ToList();
functionSequence = functionSequence.Append(new MachineCodeFunction(customShortName: "Exit")
{
Name = "Exit",
}).ToList();
// Now we reduce the function trace list by removing repeating blocks within the same func (eg. A A A B B C A A -> A B C A)
List <MachineCodeFunction> functionListBlocksJoined = new List <MachineCodeFunction>();
foreach (var function in functionSequence)
{
if (functionListBlocksJoined.Count == 0 || functionListBlocksJoined.Last() != function)
{
functionListBlocksJoined.Add(function);
}
}
string outputNodeSeq = string.Empty;
foreach (var function in functionListBlocksJoined)
{
outputNodeSeq += function.ShortName + Environment.NewLine;
}
File.WriteAllText(NODE_LIST_OUTPUT_PATH, outputNodeSeq);
string nodeStats = string.Empty;
// We also calculate how often each function is called and store that as extra info
functionListBlocksJoined.GroupBy(fun => fun)
.ToList()
.ForEach(functionCalls =>
{
var info = functionCalls.First();
info.Calls = functionCalls.Count();
nodeStats += $"Node nr. {info.ShortName} ({info.Name}), called count: {info.Calls}\r\n";
});
// Then calculate avarage and stdev, if it's more than 1 stdev color change, 2 stdev extra color change
var uniqueFunctions = functionListBlocksJoined.Distinct().ToList();
(double stdev, double avarage) = GetStandardDeviation(uniqueFunctions.Select(node => (double)node.Calls).ToList());
nodeStats = $"Avarage calls: {avarage}, standard deviation: {stdev}\r\n" + nodeStats;
foreach (var function in uniqueFunctions)
{
if (function.Calls > avarage + 2 * stdev)
{
function.NodeColor = DotColor.Green4;
nodeStats += $"Node nr. {function.ShortName} ({function.Name}), is called 2 STDEV more than AVG.\r\n";
}
else if (function.Calls > avarage + 1 * stdev)
{
function.NodeColor = DotColor.Olivedrab;
nodeStats += $"Node nr. {function.ShortName} ({function.Name}), is called 1 STDEV more than AVG.\r\n";
}
else if (function.Calls < avarage - 2 * stdev)
{
function.NodeColor = DotColor.Red1;
nodeStats += $"Node nr. {function.ShortName} ({function.Name}), is called 2 STDEV less than AVG.\r\n";
}
else if (function.Calls < avarage - 1 * stdev)
{
function.NodeColor = DotColor.Red4;
nodeStats += $"Node nr. {function.ShortName} ({function.Name}), is called 1 STDEV less than AVG.\r\n";
}
}
File.WriteAllText(NODE_STATS_OUTPUT_PATH, nodeStats);
// For each node calculate all its successors for table visualization
for (int i = 0; i < functionListBlocksJoined.Count; i++)
{
var @this = functionListBlocksJoined[i];
if (i + 1 != functionListBlocksJoined.Count)
{
@this.Next.Add(functionListBlocksJoined[i + 1]);
}
}
string outputTable = string.Empty;
var uniqueFunctionsNoRepeatingBlocks = functionListBlocksJoined.Distinct();
List <(MachineCodeFunction, List <MachineCodeFunctionGrouping>)> graphAsTable = new List <(MachineCodeFunction, List <MachineCodeFunctionGrouping>)> ();
foreach (var function in uniqueFunctionsNoRepeatingBlocks)
{
List <MachineCodeFunctionGrouping> tableEntry = function.Next.GroupBy(
functionNext => functionNext,
functionNext => functionNext,
(key, grouping) => new MachineCodeFunctionGrouping
{
ShortName = key.ShortName,
PreviousElement = function.ShortName,
NodeColor = function.NodeColor,
Key = key,
Probability = grouping.Count() / (double)function.Next.Count
}).ToList();
outputTable += function.ShortName + Environment.NewLine;
foreach (var group in tableEntry)
{
outputTable += group.ShortName + $" {Math.Round(group.Probability, 2)} ";
}
outputTable += Environment.NewLine;
graphAsTable.Add((function, tableEntry));
}
File.WriteAllText(TABLE_OUTPUT_PATH, outputTable);
// GraphViz export
var directedGraph = new DotGraph("BlockTraceGraph", true);
foreach (var(node, _) in graphAsTable)
{
var graphNode = new DotNode(node.ShortName)
{
Shape = DotNodeShape.Ellipse,
Label = node.ShortName + $"({node.Calls})",
// FillColor = node.NodeColor != null ? DotColor.Grey100 : DotColor.White,
FontColor = node.NodeColor ?? DotColor.Black,
Style = (node.NodeColor != null ? DotNodeStyle.Bold : DotNodeStyle.Default),
Height = 0.5f,
};
directedGraph.Add(graphNode);
}
foreach (var(_, edges) in graphAsTable)
{
// Let's do some coloring
// If all edges have the same weights color blue
// If there is one and ONLY one that is higher prob than everyone then GREEN
// if there is one and ONLY one that is higher prob than everyone then RED
// If only GREY
bool areAllSameProbability = edges.All(e => e.Probability == edges[0].Probability);
if (!areAllSameProbability)
{
var maxProbability = edges.Max(e => e.Probability);
var isOnly = edges.Count(e => e.Probability == maxProbability) == 1;
if (isOnly)
{
edges.First(e => e.Probability == maxProbability).Color = DotColor.Green3;
}
var minProbability = edges.Min(e => e.Probability);
var isOnlyMin = edges.Count(e => e.Probability == minProbability) == 1;
if (isOnlyMin)
{
edges.First(e => e.Probability == minProbability).Color = DotColor.Red1;
}
}
foreach (var edge in edges)
{
var arrow = new DotArrow(edge.PreviousElement, edge.ShortName)
{
ArrowLabel = Math.Round(edge.Probability, 2).ToString()
};
if (areAllSameProbability)
{
arrow.ArrowColor = DotColor.Blue;
}
if (edge.Color != null)
{
arrow.ArrowColor = edge.Color.Value;
}
if (edges.Count == 1)
{
arrow.ArrowColor = DotColor.Gray;
}
directedGraph.Add(arrow);
}
}
// Indented version
var dot = directedGraph.Compile(false);
// Save it to a file
File.WriteAllText(GRAPH_OUTPUT_PATH, dot);
}
public DotEdge(DotNode from, DotNode to)
{
this.From = from;
this.To = to;
}
private void processFunction(Function node, Function rootNode)
{
string id = node.Uid;
var currentVertex = LazyCreateNode(node);
foreach (var input in node.RootFunction.Inputs)
{
varStack.Push(input);
typeStack.Push(CntkType.Variable);
}
//add node's inputs
var inputs = node.Inputs;
for (int i = 0; i < inputs.Count; i++)
{
Variable input = inputs[i];
if (node.IsBlock && input.IsConstant)
{
continue;
}
DotNode vertex = null;
if (!input.IsOutput)
{
var name = input.Kind.ToString();
if (!string.IsNullOrEmpty(input.Name))
{
if (name.Equals("Parameter"))
{
name = input.Name;
}
else
{
name += "\n" + input.Name;
}
}
name += "\n" + ShapeDescription(input);
vertex = createVertex(input);
if (input.IsInput || input.IsPlaceholder)
{
vertex.Label = name;
//
vertex.FixedSize = true;
vertex.Height = 1f;
vertex.Width = 1.3f;
vertex.PenWidth = 4;
}
else if (string.IsNullOrEmpty(input.Name) && input.IsConstant && (input.Shape.Dimensions.Count == 0 || input.Shape.Dimensions[0] == 1))
{
string label1 = "";
var contView = new Constant(input).Value();
var value = new Value(contView);
switch (input.DataType)
{
case DataType.Float:
label1 = value.GetDenseData <float>(input)[0][0].ToString("N4", CultureInfo.InvariantCulture);
break;
case DataType.Double:
label1 = value.GetDenseData <double>(input)[0][0].ToString("N4", CultureInfo.InvariantCulture);
break;
case DataType.Float16:
label1 = (value.GetDenseData <float16>(input)[0][0]).ToString();
break;
default:
break;
}
vertex.Label = label1;
//
vertex.Height = 0.6f;
vertex.Width = 1f;
}
else
{
vertex.Label = name;
//
vertex.Height = 0.6f;
vertex.Width = 1f;
}
}
else
{
vertex = LazyCreateNode(input.Owner);
}
string label = string.IsNullOrEmpty(input.Name) ? input.Uid : input.Name;
label += "\n" + ShapeDescription(input);
DotArrow edge = new DotArrow(vertex, currentVertex);
edge.Label = label;
m_graph.Add(vertex);
m_graph.Add(currentVertex);
m_graph.Add(edge);
}
foreach (var output in node.Outputs)
{
//only final network outputs are drawn
if (node.Uid == rootNode.Uid)
{
var finalVertex = createVertex(output);
finalVertex.Label = output.Name + "\n" + ShapeDescription(output);
finalVertex.Shape = DotNodeShape.Egg;
finalVertex.FillColor = DotColor.Purple;
finalVertex.FixedSize = true;
finalVertex.Height = 1f;
finalVertex.Width = 1.3f;
finalVertex.PenWidth = 4;
//add nodes
m_graph.Add(currentVertex);
m_graph.Add(finalVertex);
}
}
//mark current node as visited
visitedNodes.Add(id);
}
private void processFunction(Function node, Function rootNode)
{
string id = node.Uid;
var currentVertex = createOutputNode(node);
foreach (var input in node.RootFunction.Inputs)
{
varStack.Push(input);
typeStack.Push(CntkType.Variable);
}
//add node's inputs
var inputs = node.Inputs;
for (int i = 0; i < inputs.Count; i++)
{
Variable input = inputs[i];
if (node.IsBlock && input.IsConstant)
{
continue;
}
DotNode vertex = null;
if (!input.IsOutput)
{
vertex = createNonOutputNode(input);
}
else
{
vertex = createOutputNode(input.Owner);
}
string label = string.IsNullOrEmpty(input.Name) ? input.Uid : input.Name;
label += "\n" + ShapeDescription(input);
DotArrow edge = new DotArrow(vertex, currentVertex);
edge.Label = label;
m_graph.Add(vertex);
m_graph.Add(currentVertex);
m_graph.Add(edge);
}
foreach (var output in node.Outputs)
{
//only final network outputs are drawn
if (node.Uid == rootNode.Uid)
{
var finalVertex = createVertex(output);
finalVertex.Label = output.Name + "\n" + ShapeDescription(output);
finalVertex.Shape = DotNodeShape.Egg;
finalVertex.FillColor = DotColor.Purple;
finalVertex.FixedSize = true;
finalVertex.Height = 1f;
finalVertex.Width = 1.3f;
finalVertex.PenWidth = 4;
//add nodes
m_graph.Add(currentVertex);
m_graph.Add(finalVertex);
}
}
//mark current node as visited
visitedNodes.Add(id);
}
public void VisitEnter(DotNode node)
{
WriteLine(node, "Dot: " + node.Name.Value);
tabs++;
}
public void Visit(DotNode node)
{
//node.Root.Accept(this);
//node.Expression.Accept(this);
node.Accept(_visitor);
}
void CreateFromJoinElement(
IASTNode path,
IASTNode alias,
int joinType,
IASTNode fetchNode,
IASTNode propertyFetch,
IASTNode with)
{
bool fetch = fetchNode != null;
if (fetch && IsSubQuery)
{
throw new QueryException("fetch not allowed in subquery from-elements");
}
// The path AST should be a DotNode, and it should have been evaluated already.
if (path.Type != DOT)
{
throw new SemanticException("Path expected for join!");
}
DotNode dot = ( DotNode )path;
//JoinType hibernateJoinType = JoinProcessor.ToHibernateJoinType( joinType );
JoinType hibernateJoinType = _impliedJoinType;
dot.JoinType = hibernateJoinType; // Tell the dot node about the join type.
dot.Fetch = fetch;
// Generate an explicit join for the root dot node. The implied joins will be collected and passed up
// to the root dot node.
dot.Resolve(true, false, alias == null ? null : alias.Text);
FromElement fromElement;
if (dot.DataType != null && dot.DataType.IsComponentType)
{
var factory = new FromElementFactory(CurrentFromClause, dot.GetLhs().FromElement, dot.PropertyPath, alias == null ? null : alias.Text, null, false);
fromElement = factory.CreateComponentJoin((ComponentType)dot.DataType);
}
else
{
fromElement = dot.GetImpliedJoin();
if (fromElement == null)
{
throw new InvalidPathException("Invalid join: " + dot.Path);
}
fromElement.SetAllPropertyFetch(propertyFetch != null);
if (with != null)
{
if (fetch)
{
throw new SemanticException("with-clause not allowed on fetched associations; use filters");
}
HandleWithFragment(fromElement, with);
}
}
if (log.IsDebugEnabled())
{
log.Debug("createFromJoinElement() : {0}", _printer.ShowAsString(fromElement, "-- join tree --"));
}
}
private (string PathDescription, int TotalPahtEffort) CalculatePathsWork(DotNode node, Dictionary <DotEdge, List <DotEdge> > dictionary)
{
return(string.Empty, 0);
}
