private static void SimulateActual(PipExecutionData data, int actualConcurrency, MultiWriter writers)
{
// simulate with actual concurrency
Console.WriteLine("Simulating actual build");
SimulationResult actualSimulation = new SimulationResult(data, data.AggregateCosts);
actualSimulation.Simulate(actualConcurrency);
Console.WriteLine("Done");
Console.WriteLine();
WriteActualAndSimulationResults(data, actualSimulation);
writers.WriteLine("Edge Count: {0}", data.DataflowGraph.EdgeCount);
writers.WriteLine("Pip Type Counts:");
foreach (var pipType in EnumTraits <PipType> .EnumerateValues())
{
writers.WriteLine("{0}: {1}", data.PipTypeCounts[(int)pipType].ToString().PadLeft(10), pipType);
}
writers.WriteLine("Processes with timing information:{0} ", data.DataflowGraph.Nodes.Where(node => data.GetPipType(node) == PipType.Process && data.StartTimes[node] != 0).Count());
writers.WriteLine("Actual Total Build Time: {0} min", data.TotalDuration.ToMinutes());
writers.WriteLine("Actual Concurrency: {0}", data.ActualConcurrency);
writers.WriteLine("Simulated total build time (using actual concurrency): {0} min", actualSimulation.TotalTime.ToMinutes());
// write down info for each critical path
ulong criticalPathCost = WriteCriticalPathToResult(writers, data, actualSimulation);
}
public Dictionary <TEnum, (long total, double ratePerSecond)> GetRates()
{
var result = new Dictionary <TEnum, (long total, double ratePerSecond)>();
lock (_activityLock)
{
ShiftWindow(_clock.UtcNow);
Snapshot?lastSnapshot = _activityWindow.LastOrDefault();
Snapshot?firstSnapshot = _activityWindow.Count > 1 ? _activityWindow.First() : null;
foreach (var v in EnumTraits <TEnum> .EnumerateValues())
{
long total = lastSnapshot?.Counters[v].Value ?? 0;
double ratePerSecond = 0;
if (lastSnapshot != null && firstSnapshot != null)
{
var duration = lastSnapshot.SnapshotTime.Subtract(firstSnapshot.SnapshotTime);
var diff = lastSnapshot.Counters[v].Value - firstSnapshot.Counters[v].Value;
ratePerSecond = ((double)diff) / duration.TotalSeconds;
}
result[v] = (total, ratePerSecond);
}
}
return(result);
}
public override Task <BoolResult> GarbageCollectAsync(OperationContext context)
{
return(context.PerformOperationAsync(Tracer,
() =>
{
// Get exclusive lock to prevent concurrent access while deleting column families
// without this, we might try to read from a column family which does not exist
return _keyValueStore.UseExclusive(
(store, db) =>
{
var otherColumnGroup = db.GetFormerColumnGroup();
store.Put(nameof(GlobalKeys.ActiveColummGroup), otherColumnGroup.ToString());
foreach (var column in EnumTraits <Columns> .EnumerateValues())
{
var columnName = db.NameOf(column, otherColumnGroup);
// Clear the column family by dropping and recreating
store.DropColumnFamily(columnName);
store.CreateColumnFamily(columnName);
}
_activeColumnsGroup = otherColumnGroup;
return BoolResult.SuccessTask;
},
this).ThrowOnError();
},
counter: Counters[ContentLocationDatabaseCounters.GarbageCollectContent],
isCritical: true));
}
/// <inheritdoc />
public override IEnumerable <(Counter, string name)> GetCounters()
{
foreach (var counterEnum in EnumTraits <TEnum> .EnumerateValues())
{
yield return(this[counterEnum], counterEnum.ToString());
}
}
private TestEnum FromInteger(ulong integerValue)
{
Contract.Assume(Enum.IsDefined(typeof(TestEnum), (int)integerValue));
TestEnum value;
bool found = EnumTraits <TestEnum> .TryConvert(integerValue, out value);
XAssert.IsTrue(found, "Failed to find a value");
return(value);
}
public void PrintCounts(MultiWriter logWriter)
{
foreach (var value in EnumTraits <TEnum> .EnumerateValues())
{
logWriter.WriteLine("{0}: {1}", value, m_counts[EnumTraits <TEnum> .ToInteger(value)]);
}
logWriter.WriteLine();
}
/// <summary>
/// Gets a counter index given an enum name.
/// </summary>
public ushort GetCounterIndex(TEnum counterId)
{
ulong counterIdValue = EnumTraits <TEnum> .ToInteger(counterId);
Contract.Assume(counterIdValue >= CounterIdOffset);
ulong relativeCounterIdValue = counterIdValue - CounterIdOffset;
return(checked ((ushort)relativeCounterIdValue));
}
/// <summary>
/// Appends <paramref name="other"/> into a current collection instance.
/// </summary>
public void Append(CounterCollection <TEnum> other)
{
foreach (var value in EnumTraits <TEnum> .EnumerateValues())
{
if (IsStopwatch(value))
{
AddToCounter(value, other.GetElapsedTime(value));
}
AddToCounter(value, other.GetCounterValue(value));
}
}
private string ToDebuggerDisplay()
{
var sb = new StringBuilder();
foreach (var enumValue in EnumTraits <TEnum> .EnumerateValues())
{
var counter = this[enumValue];
sb.AppendLine($"[{enumValue,-50}]: {counter}");
}
return(sb.ToString());
}
public void CanConvertFromInteger()
{
XAssert.AreEqual(TestEnum.A, FromInteger((ulong)TestEnum.A));
XAssert.AreEqual(TestEnum.B, FromInteger((ulong)TestEnum.B));
XAssert.AreEqual(TestEnum.C, FromInteger((ulong)TestEnum.C));
TestEnum notFound;
EnumTraits <TestEnum> .TryConvert(123, out notFound);
XAssert.AreEqual(default(TestEnum), notFound);
}
public void Set(TEnum column, object value)
{
if (value == null)
{
return;
}
var stringValue = value.ToString();
var columnIndex = EnumTraits <TEnum> .ToInteger(column);
m_maxColumnLengths[columnIndex] = Math.Max(m_maxColumnLengths[columnIndex], stringValue.Length);
m_currentRow[columnIndex] = stringValue;
}
static Inner()
{
PassThroughOperationKind = CreateOperationKind("[Pass-Through]", false);
var prefix = "PipExecutorCounter.";
foreach (var counter in EnumTraits <PipExecutorCounter> .EnumerateValues())
{
int trackedCacheCounterId = IsTrackedCacheCounter(counter) ? s_trackedCacheCounterKinds.Count : -1;
if (!CounterCollection <PipExecutorCounter> .IsStopwatch(counter))
{
s_pipExecutorCounterKinds.Add(Invalid);
continue;
}
var counterName = counter.ToString();
if (counterName.EndsWith(DurationSuffix, StringComparison.OrdinalIgnoreCase))
{
counterName = counterName.Substring(0, counterName.Length - DurationSuffix.Length);
}
counterName = prefix + counterName;
var operationKind = CreateOperationKind(counterName, /* hasPipTypeSpecialization */ IsPipTypeSpecificCounter(counter), trackedCacheCounterId: trackedCacheCounterId);
s_pipExecutorCounterKinds.Add(operationKind);
if (trackedCacheCounterId != -1)
{
s_trackedCacheCounterKinds.Add(operationKind);
}
}
prefix = "PipExecutionStep.";
foreach (var counter in EnumTraits <PipExecutionStep> .EnumerateValues())
{
var counterName = counter.ToString();
counterName = prefix + counterName;
var operationKind = CreateOperationKind(counterName, /* hasPipTypeSpecialization */ true);
s_pipExecutionStepKinds.Add(operationKind);
}
prefix = "Operation.";
foreach (var counter in EnumTraits <OperationCounter> .EnumerateValues())
{
var counterName = counter.ToString();
counterName = prefix + counterName;
var operationKind = CreateOperationKind(counterName, /* hasPipTypeSpecialization */ false);
s_operationCounterKinds.Add(operationKind);
}
}
public static ReadUsnJournalStatus ReadUsnJournalReadStatus(this ChangeJournalServiceProtocolReader reader)
{
Contract.Requires(reader != null);
var readStatusValue = reader.ReadByte();
ReadUsnJournalStatus readStatus;
if (!EnumTraits <ReadUsnJournalStatus> .TryConvert(readStatusValue, out readStatus))
{
throw new BuildXLException("Invalid ReadUsnJournalStatus");
}
return(readStatus);
}
public DisplayTable(string columnDelimeter, bool defaultHeader = true)
{
m_maxColumnLengths = new int[EnumTraits <TEnum> .ValueCount];
m_columnDelimeter = columnDelimeter;
if (defaultHeader)
{
NextRow();
foreach (var value in EnumTraits <TEnum> .EnumerateValues())
{
Set(value, value.ToString());
}
}
}
public static CounterCollection <TEnum> operator +(CounterCollection <TEnum> x, CounterCollection <TEnum> y)
{
var result = new CounterCollection <TEnum>();
foreach (var value in EnumTraits <TEnum> .EnumerateValues())
{
if (IsStopwatch(value))
{
result.AddToCounter(value, x.GetElapsedTime(value) + y.GetElapsedTime(value));
}
result.AddToCounter(value, x.GetCounterValue(value) + y.GetCounterValue(value));
}
return(result);
}
public static QueryUsnJournalResult ReadQueryUsnJournalResult(this ChangeJournalServiceProtocolReader reader)
{
Contract.Requires(reader != null);
var queryStatusValue = reader.ReadInt32();
QueryUsnJournalStatus queryStatus;
if (!EnumTraits <QueryUsnJournalStatus> .TryConvert(queryStatusValue, out queryStatus))
{
throw new BuildXLException("Invalid QueryUsnJournalStatus");
}
if (queryStatus == QueryUsnJournalStatus.Success)
{
QueryUsnJournalData data = reader.ReadUsnJournalData();
return(new QueryUsnJournalResult(queryStatus, data));
}
return(new QueryUsnJournalResult(queryStatus, data: null));
}
/// <nodoc />
public static CounterSet ToCounterSet <TEnum>(this CounterCollection <TEnum> counters)
where TEnum : struct
{
CounterSet counterSet = new CounterSet();
foreach (var counterEnum in EnumTraits <TEnum> .EnumerateValues())
{
var counter = counters[counterEnum];
var counterName = counterEnum.ToString();
counterSet.Add($"{counterName}.Count", (long)counter.Value, counter.Name);
if (counter.IsStopwatch)
{
counterSet.Add($"{counterName}.AverageMs", counter.Value != 0 ? (long)counter.Duration.TotalMilliseconds / counter.Value : 0);
counterSet.Add($"{counterName}.DurationMs", (long)counter.Duration.TotalMilliseconds);
}
}
return(counterSet);
}
/// <summary>
/// Converts <see cref="CounterCollection"/> as statistics in the form of dictionary from statistic names to values.
/// </summary>
public Dictionary <string, long> AsStatistics(string namePrefix = null)
{
Dictionary <string, long> counters = new Dictionary <string, long>();
foreach (TEnum counterName in EnumTraits <TEnum> .EnumerateValues())
{
bool isStopwatch = IsStopwatch(counterName);
string nameString = string.Format(
CultureInfo.InvariantCulture,
"{0}{1}{2}",
!string.IsNullOrWhiteSpace(namePrefix) ? namePrefix + "." : string.Empty,
counterName,
isStopwatch ? "Ms" : string.Empty);
long counterValue = isStopwatch
? (long)GetElapsedTime(counterName).TotalMilliseconds
: GetCounterValue(counterName);
counters.Add(nameString, counterValue);
}
return(counters);
}
public void TestFindByMemberValue()
{
Assert.Equal(TestEnum.Value1, EnumTraits <TestEnum> .FindByMemberValue("Value 1"));
Assert.Equal(TestEnum.Value2, EnumTraits <TestEnum> .FindByMemberValue("Value 2"));
Assert.Null(EnumTraits <TestEnum> .FindByMemberValue("other"));
}
public void Increment(TEnum value)
{
m_counts[EnumTraits <TEnum> .ToInteger(value)]++;
}
public EnumCounter()
{
m_counts = new int[EnumTraits <TEnum> .EnumerateValues().Count()];
}
static void Main(string[] args)
{
PipExecutionData data = new PipExecutionData();
if (args.Length != 1)
{
throw new ArgumentException("Specify directory path as only argument on command line");
}
string directory = args[0];
string graphPath = Path.Combine(args[0], "graph.json");
if (!File.Exists(graphPath))
{
throw new ArgumentException(string.Format("Graph does not exist at location: {0}", graphPath));
}
string resultsFormat = string.Format(@"{0}\sim\{{0}}", directory);
Console.WriteLine("Loading graph...");
Directory.CreateDirectory(string.Format(resultsFormat, ""));
data.ReadAndCacheJsonGraph(Path.Combine(directory, "graph.json"));
Console.WriteLine("Done");
Console.WriteLine();
Console.WriteLine("Saving runtime time table:");
PipRuntimeTimeTable runtimeTable = new PipRuntimeTimeTable();
foreach (var node in data.DataflowGraph.Nodes)
{
if (data.PipTypes[node] == PipType.Process)
{
var stableId = (long)data.SemiStableHashes[node];
var duration = TimeSpan.FromTicks((long)data.Durations[node]);
uint milliseconds = (uint)Math.Min(duration.TotalMilliseconds, uint.MaxValue);
runtimeTable[stableId] = new PipHistoricPerfData(PipHistoricPerfData.DefaultTimeToLive, milliseconds);
}
}
runtimeTable.SaveAsync(Path.Combine(directory, "RunningTimeTable")).Wait();
Console.WriteLine("Done");
Console.WriteLine();
using (StreamWriter sw = new StreamWriter(resultsFormat.FormatWith("result.txt")))
{
var writers = new MultiWriter(sw, Console.Out);
writers.WriteLine("Edge Count: {0}", data.DataflowGraph.EdgeCount);
writers.WriteLine("Pip Type Counts:");
foreach (var pipType in EnumTraits <PipType> .EnumerateValues())
{
writers.WriteLine("{0}: {1}", data.PipTypeCounts[(int)pipType].ToString().PadLeft(10), pipType);
}
writers.WriteLine("Processes with timing information:{0} ", data.DataflowGraph.Nodes.Where(node => data.PipTypes[node] == PipType.Process && data.StartTimes[node] != 0).Count());
File.WriteAllLines(resultsFormat.FormatWith("actual.durations.csv"), data.Spans.Where(ps => data.PipTypes[ps.Id] == PipType.Process).Select(ps => string.Join(",", data.GetName(ps.Id), ps.Duration.ToMinutes().ToString())));
File.WriteAllLines(resultsFormat.FormatWith("actual.durations.txt"), data.Spans.Select(ps => ps.Duration.ToMinutes().ToString()));
File.WriteAllLines(resultsFormat.FormatWith("actual.starts.txt"), data.Spans.Select(ps => ps.StartTime.ToMinutes().ToString()));
File.WriteAllLines(resultsFormat.FormatWith("actual.ends.txt"), data.Spans.Select(ps => ps.EndTime.ToMinutes().ToString()));
CreateSpanImage(resultsFormat.FormatWith("actual.png"), data.Spans, data.TotalDuration, Math.Max(data.ActualConcurrency, 1));
SimulationResult actualSimulation = null;
if (true)
{
Console.WriteLine("Simulating actual build");
actualSimulation = new SimulationResult(data, data.AggregateCosts);
actualSimulation.Simulate((uint)data.ActualConcurrency);
File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.durations.txt"), actualSimulation.GetSpans().Select(ps => ps.Duration.ToMinutes().ToString()));
File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.starts.txt"), actualSimulation.GetSpans().Select(ps => ps.StartTime.ToMinutes().ToString()));
File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.ends.txt"), actualSimulation.GetSpans().Select(ps => ps.EndTime.ToMinutes().ToString()));
string csvFormat = "{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}";
File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.txt"), new string[] { csvFormat.FormatWith("Id", "Thread", "Minimum Start Time", "Start Time", "End Time", "Duration", "Incoming", "Outgoing") }.Concat(actualSimulation.GetSpans().Select(ps =>
csvFormat.FormatWith(
ps.Id.Value,
ps.Thread,
actualSimulation.MinimumStartTimes[ps.Id].ToMinutes(),
ps.StartTime.ToMinutes(),
ps.EndTime.ToMinutes(),
ps.Duration.ToMinutes(),
data.DataflowGraph.GetIncomingEdgesCount(ps.Id),
data.DataflowGraph.GetIncomingEdgesCount(ps.Id)))));
CreateSimulationImage(actualSimulation, resultsFormat.FormatWith("actualSimulation.png"));
Console.WriteLine("Done");
}
Console.WriteLine();
writers.WriteLine("Actual Total Build Time: {0} min", data.TotalDuration.ToMinutes());
writers.WriteLine("Actual Concurrency: {0}", data.ActualConcurrency);
if (actualSimulation != null)
{
writers.WriteLine("Simulated total build time (using actual concurrency): {0} min", actualSimulation.TotalTime.ToMinutes());
}
int nameWidth;
int count = 0;
ulong criticalPathCost = 0;
foreach (var p in data.GetSortedPips(20, true, n => data.PipTypes[n] == PipType.Process, n => data.AggregateCosts[n]))
{
List <NodeId> criticalPath = new List <NodeId>();
NodeId criticalChainNode = p.Node;
while (criticalChainNode.IsValid())
{
criticalPath.Add(criticalChainNode);
criticalChainNode = data.CriticalChain[criticalChainNode];
}
p.Priority.Max(ref criticalPathCost);
writers.WriteLine("Critical Path {0}:", count++);
writers.WriteLine("Critical Path Cost: {0} min", p.Priority.ToMinutes());
writers.WriteLine("Critical Path Length: {0}", criticalPath.Count);
nameWidth = criticalPath.Select(n => data.GetName(n).Length).Max();
writers.WriteLine("Critical Path:\n {0}", string.Join(Environment.NewLine + " -> ", criticalPath
//.Where(n => data.Durations[n] > 0)
.Select(n => string.Format("{0} ({1} min) [{2}] <{3}>", data.GetName(n).PadLeft(nameWidth), data.Durations[n].ToMinutes(), data.PipIds[n], data.PipTypes[n]))));
}
nameWidth = data.LongestRunningPips.Select(n => data.GetName(n.Node).Length).Max();
writers.WriteLine();
writers.WriteLine("Top 20 Long Running Pips:");
foreach (var p in data.LongestRunningPips)
{
writers.WriteLine("{0} [{2}]: {1} min", data.GetName(p.Node).PadLeft(nameWidth), p.Priority.ToMinutes(), data.PipIds[p.Node]);
}
writers.WriteLine();
writers.WriteLine("Bottom 20 Shortest Running Processes:");
foreach (var p in data.ShortestRunningProcesses)
{
writers.WriteLine("{0} [{2}]: {1} min", data.GetName(p.Node).PadLeft(nameWidth), p.Priority.ToMinutes(), data.PipIds[p.Node]);
}
int simulationCount = 1;
int increment = 70;
int?[] threadCounts = new int?[simulationCount];
threadCounts[0] = 60;
//threadCounts[0] = 60;
//threadCounts[4] = 330;
//threadCounts[5] = 400;
//threadCounts[6] = 470;
SimulationResult[] results = new SimulationResult[simulationCount];
Parallel.For(0, simulationCount, i =>
{
var threadCount = threadCounts[i] ?? ((i + 1) * increment);
Console.WriteLine("Simulating {0}...", threadCount);
SimulationResult result = new SimulationResult(data, data.AggregateCosts);
result.Simulate((uint)threadCount);
results[i] = result;
Console.WriteLine("Done {0}", threadCount);
//Console.WriteLine("Simulating adjusted{0}...", threadCount);
//SimulationResult adjusted = new SimulationResult(data, data.ComputeAggregateCosts(result.GetAdjustedDurations(criticalPathCost)));
//result.Simulate((uint)threadCount);
//results[i + simulationCount] = result;
//Console.WriteLine("Done adjusted {0}", threadCount);
//i = i + (results.Length / 2);
//Console.WriteLine("Simulating {0}...", i);
//SimulationResult adjusted = new SimulationResult(data, result.EndTimes);
//adjusted.Simulate((uint)result.Threads.Count);
//results[i] = adjusted;
//Console.WriteLine("Done {0}", i);
});
results = results.OrderBy(s => s.Threads.Count).ToArray();
string format = "{0}, {1}, {5}, {6}";
writers.WriteLine();
writers.WriteLine("Results");
writers.WriteLine(format,
"Thread Count", // 0
"Total Execution Time (min)", // 1
"Critical Path Length", // 2
"Critical Path Tail Pip", // 3
"Critical Path Idle Time (min)", // 4
"Average Utilization", // 5
"Max Path Length", // 6
"Executed Pips" // 7
);
for (int i = 0; i < results.Length; i++)
{
var result = results[i];
writers.WriteLine(format,
result.Threads.Count,
result.TotalTime.ToMinutes(),
result.CriticalPath.Executions.Count,
result.CriticalPath.Executions.Last().Id.Value,
result.CriticalPath.IdleTime.ToMinutes(),
Math.Round((double)result.TotalActiveTime / (double)((ulong)result.Threads.Count * result.TotalTime), 3),
result.Threads.Select(t => t.Executions.Count).Max(),
result.Threads.Select(t => t.Executions.Count).Sum());
CreateSimulationImage(result, resultsFormat.FormatWith("simulation.{0}.png".FormatWith(result.Threads.Count)));
}
}
}
public void CanConvertToInteger()
{
XAssert.AreEqual((ulong)TestEnum.A, EnumTraits <TestEnum> .ToInteger(TestEnum.A));
XAssert.AreEqual((ulong)TestEnum.B, EnumTraits <TestEnum> .ToInteger(TestEnum.B));
XAssert.AreEqual((ulong)TestEnum.C, EnumTraits <TestEnum> .ToInteger(TestEnum.C));
}