public static IEnumerator <object> FromFile(string filename, IProgressListener progress)
{
progress.Status = "Loading diff...";
Future <string> fText;
// We could stream the lines in from the IO thread while we parse them, but this
// part of the load is usually pretty quick even on a regular hard disk, and
// loading the whole diff at once eliminates some context switches
using (var fda = new FileDataAdapter(
filename, FileMode.Open,
FileAccess.Read, FileShare.Read, 1024 * 128
)) {
var fBytes = fda.ReadToEnd();
yield return(fBytes);
fText = Future.RunInThread(
() => Encoding.ASCII.GetString(fBytes.Result)
);
yield return(fText);
}
yield return(fText);
var lr = new LineReader(fText.Result);
LineReader.Line line;
progress.Status = "Parsing diff...";
var frames = new List <TracebackFrame>();
var moduleNames = new NameTable(StringComparer.Ordinal);
var symbolTypes = new NameTable(StringComparer.Ordinal);
var functionNames = new NameTable(StringComparer.Ordinal);
var deltas = new List <DeltaInfo>();
var tracebacks = new Dictionary <UInt32, TracebackInfo>();
var regexes = new Regexes();
// Regex.Groups[string] does an inefficient lookup, so we do that lookup once here
int groupModule = regexes.DiffModule.GroupNumberFromName("module");
int groupSymbolType = regexes.DiffModule.GroupNumberFromName("symbol_type");
int groupTraceId = regexes.BytesDelta.GroupNumberFromName("trace_id");
int groupType = regexes.BytesDelta.GroupNumberFromName("type");
int groupDeltaBytes = regexes.BytesDelta.GroupNumberFromName("delta_bytes");
int groupNewBytes = regexes.BytesDelta.GroupNumberFromName("new_bytes");
int groupOldBytes = regexes.BytesDelta.GroupNumberFromName("old_bytes");
int groupNewCount = regexes.BytesDelta.GroupNumberFromName("new_count");
int groupOldCount = regexes.CountDelta.GroupNumberFromName("old_count");
int groupCountDelta = regexes.CountDelta.GroupNumberFromName("delta_count");
int groupTracebackModule = regexes.TracebackFrame.GroupNumberFromName("module");
int groupTracebackFunction = regexes.TracebackFrame.GroupNumberFromName("function");
int groupTracebackOffset = regexes.TracebackFrame.GroupNumberFromName("offset");
int groupTracebackOffset2 = regexes.TracebackFrame.GroupNumberFromName("offset2");
int groupTracebackPath = regexes.TracebackFrame.GroupNumberFromName("path");
int groupTracebackLine = regexes.TracebackFrame.GroupNumberFromName("line");
var delay = new Sleep(0.01);
int i = 0;
while (lr.ReadLine(out line))
{
retryFromHere:
i += 1;
if (i % ProgressInterval == 0)
{
progress.Maximum = lr.Length;
progress.Progress = lr.Position;
// Suspend processing for a bit
yield return(delay);
}
Match m;
if (regexes.DiffModule.TryMatch(ref line, out m))
{
moduleNames.Add(m.Groups[groupModule].Value);
}
else if (regexes.BytesDelta.TryMatch(ref line, out m))
{
var added = (m.Groups[groupType].Value == "+");
var traceId = UInt32.Parse(m.Groups[groupTraceId].Value, NumberStyles.HexNumber);
var info = new DeltaInfo {
BytesDelta = int.Parse(m.Groups[groupDeltaBytes].Value, NumberStyles.HexNumber) *
(added ? 1 : -1),
NewBytes = int.Parse(m.Groups[groupNewBytes].Value, NumberStyles.HexNumber),
OldBytes = int.Parse(m.Groups[groupOldBytes].Value, NumberStyles.HexNumber),
NewCount = int.Parse(m.Groups[groupNewCount].Value, NumberStyles.HexNumber),
};
if (lr.ReadLine(out line))
{
if (regexes.CountDelta.TryMatch(ref line, out m))
{
info.OldCount = int.Parse(m.Groups[groupOldCount].Value, NumberStyles.HexNumber);
info.CountDelta = int.Parse(m.Groups[groupCountDelta].Value, NumberStyles.HexNumber) *
(added ? 1 : -1);
}
}
bool readingLeadingWhitespace = true, doRetry = false;
frames.Clear();
var itemModules = new NameTable(StringComparer.Ordinal);
var itemFunctions = new NameTable(StringComparer.Ordinal);
while (lr.ReadLine(out line))
{
if (line.ToString().Trim().Length == 0)
{
if (readingLeadingWhitespace)
{
continue;
}
else
{
break;
}
}
else if (regexes.TracebackFrame.TryMatch(ref line, out m))
{
readingLeadingWhitespace = false;
var moduleName = moduleNames[m.Groups[groupTracebackModule].Value];
itemModules.Add(moduleName);
var functionName = functionNames[m.Groups[groupTracebackFunction].Value];
itemFunctions.Add(functionName);
var frame = new TracebackFrame {
Module = moduleName,
Function = functionName,
Offset = UInt32.Parse(m.Groups[groupTracebackOffset].Value, NumberStyles.HexNumber)
};
if (m.Groups[groupTracebackOffset2].Success)
{
frame.Offset2 = UInt32.Parse(m.Groups[groupTracebackOffset2].Value, NumberStyles.HexNumber);
}
if (m.Groups[groupTracebackPath].Success)
{
frame.SourceFile = m.Groups[groupTracebackPath].Value;
}
if (m.Groups[groupTracebackLine].Success)
{
frame.SourceLine = int.Parse(m.Groups[groupTracebackLine].Value);
}
frames.Add(frame);
}
else
{
// We hit the beginning of a new allocation, so make sure it gets parsed
doRetry = true;
break;
}
}
if (tracebacks.ContainsKey(traceId))
{
info.Traceback = tracebacks[traceId];
Program.ErrorList.ReportError("Duplicate traceback for id {0}!", traceId);
}
else
{
var frameArray = ImmutableArrayPool <TracebackFrame> .Allocate(frames.Count);
frames.CopyTo(frameArray.Array, frameArray.Offset);
info.Traceback = tracebacks[traceId] = new TracebackInfo {
TraceId = traceId,
Frames = frameArray,
Modules = itemModules,
Functions = itemFunctions
};
}
deltas.Add(info);
if (doRetry)
{
goto retryFromHere;
}
}
else if (line.StartsWith("//"))
{
// Comment, ignore it
}
else if (line.StartsWith("Total increase") || line.StartsWith("Total decrease"))
{
// Ignore this too
}
else if (line.StartsWith(" ") && (line.EndsWith(".pdb")))
{
// Symbol path for a module, ignore it
}
else
{
Program.ErrorList.ReportError("Unrecognized diff content: {0}", line.ToString());
}
}
var result = new HeapDiff(
filename, moduleNames, functionNames, deltas, tracebacks
);
yield return(new Result(result));
}
public IEnumerator <object> DiffSnapshots(HeapSnapshotInfo first, HeapSnapshotInfo last)
{
var moduleNames = new NameTable(StringComparer.Ordinal);
var heapIds = new HashSet <UInt32>();
var functionNames = new NameTable();
var deltas = new List <DeltaInfo>();
var tracebacks = new Dictionary <UInt32, TracebackInfo>();
{
var fModulesFirst = Database.SnapshotModules.Get(first.Index);
var fModulesLast = Database.SnapshotModules.Get(last.Index);
var fHeapsFirst = Database.SnapshotHeaps.Get(first.Index);
var fHeapsLast = Database.SnapshotHeaps.Get(last.Index);
yield return(fModulesFirst);
foreach (var moduleName in fModulesFirst.Result)
{
moduleNames.Add(Path.GetFileNameWithoutExtension(moduleName));
}
yield return(fHeapsFirst);
heapIds.UnionWith(from heap in fHeapsFirst.Result select heap.HeapID);
yield return(fModulesLast);
foreach (var moduleName in fModulesLast.Result)
{
moduleNames.Add(Path.GetFileNameWithoutExtension(moduleName));
}
yield return(fHeapsLast);
heapIds.UnionWith(from heap in fHeapsLast.Result select heap.HeapID);
}
var allocationIds = new HashSet <UInt32>();
{
var fAllocations = Database.HeapAllocations.Select(heapIds);
using (fAllocations)
yield return(fAllocations);
yield return(Future.RunInThread(() => {
foreach (var ids in fAllocations.Result)
{
allocationIds.UnionWith(ids);
}
}));
}
{
var tracebackIds = new HashSet <UInt32>();
var deallocs = new Dictionary <UInt32, DeltaInfo>();
var allocs = new Dictionary <UInt32, DeltaInfo>();
var fAllocationRanges = Database.Allocations.Select(allocationIds);
using (fAllocationRanges)
yield return(fAllocationRanges);
yield return(Future.RunInThread(() => {
DeltaInfo delta;
foreach (var item in fAllocationRanges.Result)
{
var ranges = item.Ranges.Array;
for (int i = 0, c = item.Ranges.Count, o = item.Ranges.Offset; i < c; i++)
{
var range = ranges[i + o];
if ((range.First <= first.Index) &&
(range.Last >= first.Index) &&
(range.Last < last.Index)
)
{
// deallocation
if (deallocs.TryGetValue(range.TracebackID, out delta))
{
delta.CountDelta += 1;
delta.BytesDelta += (int)(range.Size + range.Overhead);
delta.OldCount += 1;
delta.OldBytes += (int)(range.Size + range.Overhead);
}
else
{
deallocs.Add(range.TracebackID, new DeltaInfo {
Added = false,
BytesDelta = (int)(range.Size + range.Overhead),
CountDelta = 1,
NewBytes = 0,
NewCount = 0,
OldBytes = (int)(range.Size + range.Overhead),
OldCount = 1,
TracebackID = range.TracebackID,
Traceback = null
});
}
tracebackIds.Add(range.TracebackID);
}
else if (
(range.First <= last.Index) &&
(range.First > first.Index) &&
(range.Last >= last.Index)
)
{
// allocation
if (allocs.TryGetValue(range.TracebackID, out delta))
{
delta.CountDelta += 1;
delta.BytesDelta += (int)(range.Size + range.Overhead);
delta.NewCount += 1;
delta.NewBytes += (int)(range.Size + range.Overhead);
}
else
{
allocs.Add(range.TracebackID, new DeltaInfo {
Added = true,
BytesDelta = (int)(range.Size + range.Overhead),
CountDelta = 1,
NewBytes = (int)(range.Size + range.Overhead),
NewCount = 1,
OldBytes = 0,
OldCount = 0,
TracebackID = range.TracebackID,
Traceback = null
});
}
tracebackIds.Add(range.TracebackID);
}
}
}
foreach (var tracebackId in tracebackIds)
{
if (allocs.TryGetValue(tracebackId, out delta))
{
DeltaInfo dealloc;
if (deallocs.TryGetValue(tracebackId, out dealloc))
{
delta.OldBytes = dealloc.OldBytes;
delta.OldCount = dealloc.OldCount;
delta.BytesDelta = Math.Abs(delta.NewBytes - delta.OldBytes);
delta.CountDelta = Math.Abs(delta.NewCount - delta.OldCount.Value);
if (delta.NewBytes < delta.OldBytes)
{
delta.Added = false;
}
}
if (delta.BytesDelta != 0)
{
deltas.Add(delta);
}
}
else if (deallocs.TryGetValue(tracebackId, out delta))
{
if (delta.BytesDelta != 0)
{
deltas.Add(delta);
}
}
}
}));
var fTracebacks = Database.Tracebacks.Select(tracebackIds);
using (fTracebacks)
yield return(fTracebacks);
Dictionary <UInt32, TracebackFrame> frameSymbols;
{
var rawFrames = new HashSet <UInt32>();
yield return(Future.RunInThread(() => {
foreach (var traceback in fTracebacks.Result)
{
foreach (var rawFrame in traceback)
{
rawFrames.Add(rawFrame);
}
}
}));
var fSymbols = Database.SymbolCache.Select(rawFrames);
using (fSymbols)
yield return(fSymbols);
var fSymbolDict = Future.RunInThread(() =>
SequenceUtils.ToDictionary(rawFrames, fSymbols.Result)
);
yield return(fSymbolDict);
frameSymbols = fSymbolDict.Result;
}
yield return(Future.RunInThread(() => {
foreach (var tf in frameSymbols.Values)
{
if (tf.Function != null)
{
functionNames.Add(tf.Function);
}
}
foreach (var traceback in fTracebacks.Result)
{
tracebacks[traceback.ID] = ConstructTracebackInfo(
traceback.ID, traceback.Frames, frameSymbols
);
}
foreach (var d in deltas)
{
d.Traceback = tracebacks[d.TracebackID];
}
}));
}
yield return(Future.RunInThread(() =>
deltas.Sort((lhs, rhs) => {
var lhsBytes = (lhs.Added ? 1 : -1) * lhs.BytesDelta;
var rhsBytes = (rhs.Added ? 1 : -1) * rhs.BytesDelta;
return rhsBytes.CompareTo(lhsBytes);
})
));
yield return(Result.New(new HeapDiff(
null, moduleNames, functionNames, deltas, tracebacks
)));
}
