TimingData SummarizeParsedTimingData(string SummaryName, TimingDataType TimingType, IEnumerable <string> Lines)
{
TimingData Summary = new TimingData()
{
Name = SummaryName, Type = TimingDataType.Summary
};
List <TimingData> ParsedTimingData = ParseTimingDataFromLines(TimingType, Lines);
foreach (TimingData Data in ParsedTimingData)
{
// See if we've already added a child that matches this data's name. If so, just add to the duration.
TimingData MatchedData;
if (Summary.Children.TryGetValue(Data.Name, out MatchedData))
{
MatchedData.ExclusiveDuration += Data.ExclusiveDuration;
}
else
{
Summary.AddChild(Data);
}
}
// Exclusive duration was originally set to the inclusive value, remove the children's durations to get the
// actual exclusive duration.
Summary.ExclusiveDuration = Summary.ExclusiveDuration - Summary.Children.Sum(c => c.Value.InclusiveDuration);
return(Summary);
}
TimingData SummarizeParsedTimingData(string SummaryName, TimingDataType TimingType, IEnumerable <string> Lines)
{
TimingData Summary = new TimingData()
{
Name = SummaryName, Type = TimingDataType.Summary
};
List <TimingData> ParsedTimingData = ParseTimingDataFromLines(TimingType, Lines);
foreach (TimingData Data in ParsedTimingData)
{
// See if we've already added a child that matches this data's name. If so, just add to the duration.
TimingData MatchedData;
if (Summary.Children.TryGetValue(Data.Name, out MatchedData))
{
MatchedData.Count += 1;
MatchedData.ExclusiveDuration += Data.ExclusiveDuration;
}
else
{
Summary.AddChild(Data);
}
}
return(Summary);
}
public override int Execute(CommandLineArguments Arguments)
{
FileReference ManifestFile = Arguments.GetFileReference("-ManifestFile=");
string[] ParsedFileNames = FileReference.ReadAllLines(ManifestFile);
// Load all the file timing data and summarize them for the aggregate.
FileTimingData = new TimingData()
{
Name = "Files", Type = TimingDataType.Summary
};
Parallel.ForEach(ParsedFileNames, new ParallelOptions()
{
MaxDegreeOfParallelism = 4
}, ParseTimingDataFile);
// Create aggregate summary. Duration is the duration of the files in the aggregate.
string AggregateName = Arguments.GetString("-Name=");
TimingData AggregateData = new TimingData()
{
Name = AggregateName, Type = TimingDataType.Aggregate
};
AggregateData.AddChild(FileTimingData);
// Group the includes, classes, and functions by name and sum them up then add to the aggregate.
GroupTimingDataOnName(AggregateData, "Include Timings", AggregateIncludes);
GroupTimingDataOnName(AggregateData, "Class Timings", AggregateClasses);
GroupTimingDataOnName(AggregateData, "Function Timings", AggregateFunctions);
// Write out aggregate summary.
string OutputFile = Path.Combine(ManifestFile.Directory.FullName, String.Format("{0}.timing.bin", AggregateName));
using (BinaryWriter Writer = new BinaryWriter(File.Open(OutputFile, FileMode.Create)))
{
// Write out the aggregate data.
Writer.Write(AggregateData);
// Write the look up table for the compressed binary blobs.
int Offset = 0;
Writer.Write(CompressedFiles.Count);
foreach (KeyValuePair <string, byte[]> CompressedFile in CompressedFiles)
{
Writer.Write(CompressedFile.Key);
Writer.Write(Offset);
Writer.Write(CompressedFile.Value.Length);
Writer.Write(DecompressedFileSizes[CompressedFile.Key]);
Offset += CompressedFile.Value.Length;
}
// Write the compressed binary blobs.
foreach (KeyValuePair <string, byte[]> CompressedFile in CompressedFiles)
{
Writer.Write(CompressedFile.Value);
}
}
return(0);
}
/// <summary>
/// Constructs a new instance of <see cref="TimingData"/> using data from a <see cref="BinaryReader"/>.
/// </summary>
/// <param name="Reader">The <see cref="BinaryReader"/> to get the data from.</param>
public TimingData(BinaryReader Reader)
{
Name = Reader.ReadString();
Type = (TimingDataType)Reader.ReadByte();
Count = Reader.ReadInt32();
ExclusiveDuration = Reader.ReadDouble();
int ChildCount = Reader.ReadInt32();
for (int i = 0; i < ChildCount; ++i)
{
TimingData NewChild = new TimingData(Reader);
Children.Add(NewChild.Name, NewChild);
}
}
List <TimingData> ParseTimingDataFromLines(TimingDataType TimingType, IEnumerable <string> Lines)
{
List <TimingData> ParsedTimingData = new List <TimingData>();
int LastDepth = 0;
TimingData LastTimingData = null;
foreach (string Line in Lines)
{
int LineDepth;
TimingData CurrentTimingData = ParseTimingDataFromLine(TimingType, Line, out LineDepth);
if (LineDepth == 0)
{
ParsedTimingData.Add(CurrentTimingData);
}
else
{
while (LineDepth < LastDepth)
{
LastTimingData = LastTimingData.Parent;
--LastDepth;
}
// If this timing data would have a parent, add the data to that parent and reduce its exclusive
// duration by this data's inclusive duration.
TimingData ParentData = null;
if (LineDepth == LastDepth)
{
CurrentTimingData.Parent = LastTimingData.Parent;
ParentData = LastTimingData.Parent;
}
else if (LineDepth > LastDepth)
{
CurrentTimingData.Parent = LastTimingData;
ParentData = LastTimingData;
}
if (ParentData != null)
{
ParentData.AddChild(CurrentTimingData);
ParentData.ExclusiveDuration -= CurrentTimingData.InclusiveDuration;
}
}
LastTimingData = CurrentTimingData;
LastDepth = LineDepth;
}
return(ParsedTimingData);
}
/// <summary>
/// Creates a deep clone of this event.
/// </summary>
/// <returns>A deep clone of this event.</returns>
public TimingData Clone()
{
TimingData ClonedTimingData = new TimingData()
{
Name = Name,
Type = Type,
ExclusiveDuration = ExclusiveDuration,
};
foreach (KeyValuePair <string, TimingData> Child in Children)
{
ClonedTimingData.Children.Add(Child.Key, Child.Value.Clone());
}
return(ClonedTimingData);
}
/// <summary>
/// Adds a sub-event (child) to this event if it doesn't exist, or increase the duration by the
/// duration of the provided event.
/// </summary>
/// <param name="ChildData">The sub-event to add to this event.</param>
public void AddChild(TimingData ChildData)
{
TimingData MatchingData;
if (Children.TryGetValue(ChildData.Name, out MatchingData))
{
MatchingData.ExclusiveDuration += ChildData.ExclusiveDuration;
foreach (TimingData ChildChildData in ChildData.Children.Values)
{
MatchingData.AddChild(ChildChildData);
}
}
else
{
Children.Add(ChildData.Name, ChildData);
}
}
private void GroupTimingDataOnName(TimingData AggregateData, string TimingName, IEnumerable <TimingData> UngroupedData, int NumberToKeep = int.MaxValue)
{
string GroupName = String.Format("Top {0} {1}", NumberToKeep, TimingName);
TimingData GroupedTimingData = new TimingData()
{
Name = GroupName, Type = TimingDataType.Summary
};
IEnumerable <IGrouping <string, TimingData> > Groups = UngroupedData.GroupBy(i => i.Name).OrderByDescending(g => g.Sum(d => d.ExclusiveDuration)).ToList();
foreach (IGrouping <string, TimingData> Group in Groups.Take(NumberToKeep))
{
TimingData GroupedData = new TimingData()
{
Name = Group.Key, ExclusiveDuration = Group.Sum(d => d.ExclusiveDuration)
};
GroupedTimingData.Children.Add(Group.Key, GroupedData);
}
AggregateData.AddChild(GroupedTimingData);
}
private void SummarizeTimingData(IEnumerable <TimingData> Data, TimingData Summary, string TypeName, int NumberToKeep)
{
// Sort the data by inclusive duration, highest first, so we can grab the top entries to keep.
List <TimingData> Sorted = Data.OrderByDescending(v => v.InclusiveDuration).ToList();
Summary.Name = String.Format("Total {0}: {1}", TypeName, Sorted.Count);
IEnumerable <TimingData> TopData = Sorted.Take(NumberToKeep);
foreach (TimingData Child in TopData)
{
TimingData SummarizedChild = new TimingData()
{
Name = Child.Name, Parent = Summary, Type = Child.Type, ExclusiveDuration = Child.InclusiveDuration
};
Summary.AddChild(SummarizedChild);
}
// Set the summary to the duration of all children, not just the top ten.
Summary.ExclusiveDuration = Sorted.Except(TopData).Sum(c => c.InclusiveDuration);
}
private void GroupTimingDataOnName(TimingData AggregateData, string TimingName, IEnumerable <TimingData> UngroupedData)
{
string GroupName = TimingName;
TimingData GroupedTimingData = new TimingData()
{
Name = GroupName, Type = TimingDataType.Summary
};
IEnumerable <IGrouping <string, TimingData> > Groups = UngroupedData.GroupBy(i => i.Name).OrderByDescending(g => g.Sum(d => d.ExclusiveDuration)).ToList();
foreach (IGrouping <string, TimingData> Group in Groups)
{
TimingData GroupedData = new TimingData()
{
Name = Group.Key, ExclusiveDuration = Group.Sum(d => d.ExclusiveDuration), Count = Group.Sum(d => d.Count)
};
GroupedTimingData.Children.Add(Group.Key, GroupedData);
}
AggregateData.AddChild(GroupedTimingData);
AggregateData.ExclusiveDuration -= GroupedTimingData.InclusiveDuration;
}
TimingData ParseTimingDataFromLine(TimingDataType TimingType, string Line, out int LineDepth)
{
Match TimingDataMatch = Regex.Match(Line, TimingDataRegex);
if (!TimingDataMatch.Success)
{
LineDepth = -1;
return(null);
}
LineDepth = TimingDataMatch.Groups["Indent"].Success ? TimingDataMatch.Groups["Indent"].Value.Count() : 0;
TimingData ParsedTimingData = new TimingData()
{
Name = TimingDataMatch.Groups["Name"].Value,
Type = TimingType,
ExclusiveDuration = float.Parse(TimingDataMatch.Groups["Duration"].Value),
};
return(ParsedTimingData);
}
private IEnumerable <TimingData> GroupChildren(IEnumerable <TimingData> Children, TimingDataType Type)
{
List <TimingData> GroupedChildren = new List <TimingData>();
Dictionary <string, List <TimingData> > ChildGroups = new Dictionary <string, List <TimingData> >();
foreach (TimingData Child in Children)
{
// See if this is a templated class. If not, add it as is.
Match Match = Regex.Match(Child.Name, @"^([^<]*)(?<Template><.*>)");
if (!Match.Success)
{
GroupedChildren.Add(Child);
}
else
{
// Generate group name from template.
int TemplateParamCount = Match.Groups["Template"].Value.Count(c => c == ',') + 1;
List <string> TemplateParamSig = new List <string>(TemplateParamCount);
for (int i = 0; i < TemplateParamCount; ++i)
{
TemplateParamSig.Add("...");
}
string GroupName = Child.Name.Replace(Match.Groups["Template"].Value, String.Format("<{0}>", string.Join(", ", TemplateParamSig)));
// See if we have a group for this template already. If not, add it.
if (!ChildGroups.ContainsKey(GroupName))
{
ChildGroups.Add(GroupName, new List <TimingData>());
}
ChildGroups[GroupName].Add(Child);
}
}
// Add grouped children.
foreach (KeyValuePair <string, List <TimingData> > Group in ChildGroups)
{
if (Group.Value.Count == 1)
{
GroupedChildren.Add(Group.Value.First());
continue;
}
TimingData NewViewModel = new TimingData()
{
Name = Group.Key,
Type = Type,
};
foreach (TimingData Child in Group.Value)
{
TimingData NewChild = new TimingData()
{
Name = Child.Name, Type = Child.Type, Parent = NewViewModel, ExclusiveDuration = Child.ExclusiveDuration
};
NewViewModel.AddChild(NewChild);
}
GroupedChildren.Add(NewViewModel);
}
return(GroupedChildren);
}
private void ParseTimingDataFile(string ParsedFileName)
{
// Convert input file back into summary objects.
using (BinaryReader Reader = new BinaryReader(File.Open(ParsedFileName, FileMode.Open, FileAccess.Read)))
{
TimingData ParsedTimingData = new TimingData(Reader);
Task CompressDataTask = Task.Run(() =>
{
Reader.BaseStream.Seek(0, SeekOrigin.Begin);
using (MemoryStream CompressedMemoryStream = new MemoryStream())
{
using (GZipStream CompressionStream = new GZipStream(CompressedMemoryStream, CompressionMode.Compress))
{
Reader.BaseStream.CopyTo(CompressionStream);
}
CompressedFiles.TryAdd(ParsedTimingData.Name, CompressedMemoryStream.ToArray());
DecompressedFileSizes.TryAdd(ParsedTimingData.Name, (int)Reader.BaseStream.Length);
}
});
TimingData SummarizedTimingData = new TimingData()
{
Name = ParsedTimingData.Name, Parent = FileTimingData, Type = TimingDataType.Summary
};
foreach (TimingData Include in ParsedTimingData.Children["IncludeTimings"].Children.Values)
{
SummarizedTimingData.AddChild(Include.Clone());
}
SummarizedTimingData.ExclusiveDuration += ParsedTimingData.Children["ClassTimings"].Children.Values.Sum(d => d.InclusiveDuration);
SummarizedTimingData.ExclusiveDuration += ParsedTimingData.Children["FunctionTimings"].Children.Values.Sum(d => d.InclusiveDuration);
// Gather and update child timing data.
Task QueueIncludesTask = Task.Run(() =>
{
// Flatten the includes for aggregation.
IEnumerable <TimingData> Includes = ParsedTimingData.Children["IncludeTimings"].Children.Values;
Dictionary <string, TimingData> FlattenedIncludes = new Dictionary <string, TimingData>();
FlattenIncludes(FlattenedIncludes, Includes);
foreach (TimingData Include in FlattenedIncludes.Values)
{
AggregateIncludes.Add(Include);
}
});
Task QueueClassesTask = Task.Run(() =>
{
// Collapse templates into single entries.
IEnumerable <TimingData> CollapsedClasses = GroupChildren(ParsedTimingData.Children["ClassTimings"].Children.Values, TimingDataType.Class);
foreach (TimingData Class in CollapsedClasses)
{
AggregateClasses.Add(new TimingData()
{
Name = Class.Name, Type = TimingDataType.Class, ExclusiveDuration = Class.InclusiveDuration
});
}
});
Task QueueFunctionsTask = Task.Run(() =>
{
// Collapse templates into single entries.
IEnumerable <TimingData> CollapsedFunctions = GroupChildren(ParsedTimingData.Children["FunctionTimings"].Children.Values, TimingDataType.Function);
foreach (TimingData Function in CollapsedFunctions)
{
AggregateFunctions.Add(new TimingData()
{
Name = Function.Name, Type = TimingDataType.Function, ExclusiveDuration = Function.InclusiveDuration
});
}
});
while (!CompressDataTask.IsCompleted || !QueueIncludesTask.IsCompleted || !QueueClassesTask.IsCompleted || !QueueFunctionsTask.IsCompleted)
{
Thread.Sleep(TimeSpan.FromMilliseconds(50));
}
lock (AddFileLock)
{
FileTimingData.AddChild(SummarizedTimingData);
}
}
}
public override int Execute(CommandLineArguments Arguments)
{
FileReference InputFile = Arguments.GetFileReference("-TimingFile=");
// If the tracing argument was passed, hand off to the logic to generate a JSON file compatible with
// chrome://tracing
if (Arguments.HasOption("-Tracing"))
{
ParseTimingDataToTracingFiles(InputFile);
return(0);
}
// Break the input file into the various sections for processing.
string[] AllLines = FileReference.ReadAllLines(InputFile);
List <string> Includes = new List <string>();
List <string> Classes = new List <string>();
List <string> Functions = new List <string>();
TimingDataType CurrentType = TimingDataType.None;
foreach (string Line in AllLines)
{
if (string.IsNullOrWhiteSpace(Line))
{
continue;
}
// Check for a change of type.
if (Line.StartsWith("Include Headers:", StringComparison.OrdinalIgnoreCase))
{
CurrentType = TimingDataType.Include;
continue;
}
else if (Line.StartsWith("Class Definitions:", StringComparison.OrdinalIgnoreCase))
{
CurrentType = TimingDataType.Class;
continue;
}
else if (Line.StartsWith("Function Definitions:", StringComparison.OrdinalIgnoreCase))
{
CurrentType = TimingDataType.Function;
continue;
}
// Skip the count line, we don't need it.
if (Regex.IsMatch(Line, @"^\tCount\:\s*\d*$"))
{
continue;
}
// If we didn't change types and this isn't the count line and it doesn't match the expected output,
// clear the current type and move on.
Match TimingDataMatch = Regex.Match(Line, TimingDataRegex);
if (!TimingDataMatch.Success)
{
CurrentType = TimingDataType.None;
continue;
}
// If we get to here this is a line we want to parse. Add it to the correct collection.
switch (CurrentType)
{
case TimingDataType.Include:
{
Includes.Add(Line);
break;
}
case TimingDataType.Class:
{
Classes.Add(Line);
break;
}
case TimingDataType.Function:
{
Functions.Add(Line);
break;
}
}
}
// Build the summary.
TimingData Summary = new TimingData()
{
Name = InputFile.FullName.Replace(".timing.txt", string.Empty), Type = TimingDataType.Summary
};
Summary.AddChild(SummarizeParsedTimingData("IncludeTimings", TimingDataType.Include, Includes));
Summary.AddChild(SummarizeParsedTimingData("ClassTimings", TimingDataType.Class, Classes));
Summary.AddChild(SummarizeParsedTimingData("FunctionTimings", TimingDataType.Function, Functions));
// Write out the timing binary file.
using (BinaryWriter Writer = new BinaryWriter(File.Open(InputFile.ChangeExtension(".timing.bin").FullName, FileMode.Create)))
{
Writer.Write(Summary);
}
return(0);
}
