/// <summary>
/// Creates a new ConsoleMultiWriter to copy or redirect console output to file(s)
/// </summary>
/// <param name="writeToConsole">If you wish to continue writing output to the console</param>
/// <param name="outputPaths"></param>
/// <param name="timestamps">If you wish for DateTime stamps to be inserted on new file lines</param>
/// <param name="replaceFiles">If the output files should be replaced, if they exist</param>
/// <param name="combineErrorOutput">Combine the output of Console and Console.Error into the Console files(s)</param>
/*public ConsoleMultiWriter(bool writeToConsole, string[] outputPaths, bool timestamps = false, bool replaceFiles = false, bool combineErrorOutput = true)
* {
* if (outputPaths.Length == 0)
* {
* throw new InvalidOperationException("Must have at least one outputPath for ConsoleMultiWriter. None where provided.");
* }
*
* // Maintain easy reference to original output
* originalOutput = Console.Out;
* originalErrorOutput = Console.Error;
*
* try
* {
* (fileStreams, fileWriterStreams) = CreateFileWriters(replaceFiles, outputPaths);
* fileWriter = new MultiWriter(timestamps, fileWriterStreams.ToArray());
*
* // If we want to write to the console
* // include the original TextWriter in the multiwriter
* if (writeToConsole)
* {
* multiWriter = new MultiWriter(timestamps, originalOutput, fileWriterStreams.ToArray());
* }
* else
* {
* multiWriter = new MultiWriter(timestamps, fileWriterStreams.ToArray());
* }
* }
* catch (Exception e)
* {
* Console.WriteLine("Cannot open file for writing");
* Console.WriteLine(e.Message);
* return;
* }
* Console.SetOut(multiWriter);
*
* if (combineErrorOutput)
* {
* Console.SetError(multiWriter);
* }
* }*/
/// <summary>
/// Creates a new ConsoleMultiWriter to copy or redirect console output to file(s)
/// </summary>
/// <param name="writeToConsole">If you wish to continue writing output to the console</param>
/// <param name="errorOutputPath">The output path for just errors if Console.Error is to be redirected</param>
/// <param name="timestamps">If you wish for DateTime stamps to be inserted on new file lines</param>
/// <param name="replaceFiles">If the output files should be replaced, if they exist</param>
/// <param name="combineErrorOutput">Combine the output of Console and Console.Error into the normal Console files(s)</param>
/// <param name="outputPaths"></param>
public ConsoleMultiWriter(bool writeToConsole, string errorOutputPath, string[] outputPaths, bool timestamps = false, bool replaceFiles = false, bool combineErrorOutput = true)
{
if (outputPaths is null)
{
throw new ArgumentNullException("outputPaths cannot be null");
}
// Ensure there are no duplicate paths, should this ban error instead?
string[] distinctOutputPaths = EnsurePathsAreUnique(outputPaths);
// Maintain easy reference to original output
originalOutput = Console.Out;
originalErrorOutput = Console.Error;
// Writer that will write messages to normal console and outputs
MultiWriter multiWriter = CreateMultiWriter(replaceFiles, timestamps, distinctOutputPaths, writeToConsole ? originalOutput : null);
fileWriter = CreateMultiWriter(replaceFiles, timestamps, multiWriter.FileWriters); //Writes Console only to files
// Writer that will write Console.Error's to error file, console, and to the normal multiWriter
MultiWriter errorMultiWriter = CreateMultiWriter(replaceFiles, timestamps, new[] { errorOutputPath }, multiWriter);
fileErrorWriter = CreateMultiWriter(replaceFiles, timestamps, errorMultiWriter.FileWriters, multiWriter); //Writes Console.Error only to files
Console.SetOut(multiWriter);
Console.SetError(errorMultiWriter);
}
private IWriter OpenOut(string expression)
{
if (expression.Contains("|"))
{
var multiWriter = new MultiWriter();
foreach (var subExpression in expression.Split('|'))
{
multiWriter.Add(this.OpenOut(subExpression.Trim()));
}
return(multiWriter);
}
if ("<std>".Equals(expression.ToLower()))
{
return(new ConsoleWriter());
}
if ("<off>".Equals(expression.ToLower()))
{
return(new EmptyWriter());
}
return(new FileWriter(expression.Trim()));
}
private MultiWriter CreateMultiWriter(bool replaceFiles, bool timestamps, string[] paths, TextWriter consoleOutput = null)
{
(List <FileStream> streams, List <StreamWriter> writers) = CreateFileWriters(replaceFiles, paths);
MultiWriter fileWriter = CreateMultiWriter(replaceFiles, timestamps, writers, consoleOutput);
return(fileWriter);
}
public void PrintCounts(MultiWriter logWriter)
{
foreach (var value in EnumTraits <TEnum> .EnumerateValues())
{
logWriter.WriteLine("{0}: {1}", value, m_counts[EnumTraits <TEnum> .ToInteger(value)]);
}
logWriter.WriteLine();
}
/// <summary>
/// Constructor
/// </summary>
public LoggerControl()
{
InitializeComponent();
FormatWriter format = new FormatWriter(new ListBoxWriter(lbLog), new DetailFormat());
_threadWriter = new ThreadWriter(format);
_writer = new MultiWriter
{
new ContainsWriter(_threadWriter, _MARKER, true),
new ContainsWriter(format, _MARKER, false)
};
}
private MultiWriter CreateMultiWriter(bool replaceFiles, bool timestamps, IEnumerable <StreamWriter> fileWriters, MultiWriter nestedWriter, TextWriter consoleOutput = null)
{
MultiWriter fileWriter;
if (consoleOutput is null)
{
fileWriter = new MultiWriter(timestamps, nestedWriter, fileWriters.ToArray());
}
else
{
fileWriter = new MultiWriter(timestamps, consoleOutput, nestedWriter, fileWriters.ToArray());
}
return(fileWriter);
}
private MultiWriter CreateMultiWriter(bool replaceFiles, bool timestamps, string[] paths, MultiWriter nestedWriter, TextWriter consoleOutput = null)
{
(List <FileStream> streams, List <StreamWriter> writers) = CreateFileWriters(replaceFiles, paths);
MultiWriter fileWriter;
if (consoleOutput is null)
{
fileWriter = new MultiWriter(timestamps, nestedWriter, writers.ToArray());
}
else
{
fileWriter = new MultiWriter(timestamps, consoleOutput, nestedWriter, writers.ToArray());
}
return(fileWriter);
}
public void Dispose()
{
Console.SetOut(originalOutput);
Console.SetError(originalErrorOutput);
foreach (var fileWriter in fileWriterStreams)
{
fileWriter.Flush();
fileWriter.Close();
}
foreach (var fileStream in fileStreams)
{
try
{
fileStream.Flush();
fileStream.Close();
}
catch
{
continue;
}
}
fileWriter?.Dispose();
fileErrorWriter?.Dispose();
multiWriter?.Dispose();
multiErrorWriter?.Dispose();
fileWriter = null;
fileErrorWriter = null;
multiWriter = null;
multiErrorWriter = null;
fileWriterStreams.Clear();
fileStreams.Clear();
fileWriterStreams = null;
fileStreams = null;
}
static void Main(string[] args)
{
PipExecutionData data = new PipExecutionData();
if (args.Length != 1)
{
throw new ArgumentException("Specify directory path as only argument on command line");
}
string directory = args[0];
string graphPath = Path.Combine(args[0], "graph.json");
if (!File.Exists(graphPath))
{
throw new ArgumentException(string.Format("Graph does not exist at location: {0}", graphPath));
}
string resultsFormat = string.Format(@"{0}\sim\{{0}}", directory);
Console.WriteLine("Loading graph...");
Directory.CreateDirectory(string.Format(resultsFormat, ""));
data.ReadAndCacheJsonGraph(Path.Combine(directory, "graph.json"));
Console.WriteLine("Done");
Console.WriteLine();
Console.WriteLine("Saving runtime time table:");
PipRuntimeTimeTable runtimeTable = new PipRuntimeTimeTable();
foreach (var node in data.DataflowGraph.Nodes)
{
if (data.PipTypes[node] == PipType.Process)
{
var stableId = (long)data.SemiStableHashes[node];
var duration = TimeSpan.FromTicks((long)data.Durations[node]);
uint milliseconds = (uint)Math.Min(duration.TotalMilliseconds, uint.MaxValue);
runtimeTable[stableId] = new PipHistoricPerfData(PipHistoricPerfData.DefaultTimeToLive, milliseconds);
}
}
runtimeTable.SaveAsync(Path.Combine(directory, "RunningTimeTable")).Wait();
Console.WriteLine("Done");
Console.WriteLine();
using (StreamWriter sw = new StreamWriter(resultsFormat.FormatWith("result.txt")))
{
var writers = new MultiWriter(sw, Console.Out);
writers.WriteLine("Edge Count: {0}", data.DataflowGraph.EdgeCount);
writers.WriteLine("Pip Type Counts:");
foreach (var pipType in EnumTraits <PipType> .EnumerateValues())
{
writers.WriteLine("{0}: {1}", data.PipTypeCounts[(int)pipType].ToString().PadLeft(10), pipType);
}
writers.WriteLine("Processes with timing information:{0} ", data.DataflowGraph.Nodes.Where(node => data.PipTypes[node] == PipType.Process && data.StartTimes[node] != 0).Count());
File.WriteAllLines(resultsFormat.FormatWith("actual.durations.csv"), data.Spans.Where(ps => data.PipTypes[ps.Id] == PipType.Process).Select(ps => string.Join(",", data.GetName(ps.Id), ps.Duration.ToMinutes().ToString())));
File.WriteAllLines(resultsFormat.FormatWith("actual.durations.txt"), data.Spans.Select(ps => ps.Duration.ToMinutes().ToString()));
File.WriteAllLines(resultsFormat.FormatWith("actual.starts.txt"), data.Spans.Select(ps => ps.StartTime.ToMinutes().ToString()));
File.WriteAllLines(resultsFormat.FormatWith("actual.ends.txt"), data.Spans.Select(ps => ps.EndTime.ToMinutes().ToString()));
CreateSpanImage(resultsFormat.FormatWith("actual.png"), data.Spans, data.TotalDuration, Math.Max(data.ActualConcurrency, 1));
SimulationResult actualSimulation = null;
if (true)
{
Console.WriteLine("Simulating actual build");
actualSimulation = new SimulationResult(data, data.AggregateCosts);
actualSimulation.Simulate((uint)data.ActualConcurrency);
File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.durations.txt"), actualSimulation.GetSpans().Select(ps => ps.Duration.ToMinutes().ToString()));
File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.starts.txt"), actualSimulation.GetSpans().Select(ps => ps.StartTime.ToMinutes().ToString()));
File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.ends.txt"), actualSimulation.GetSpans().Select(ps => ps.EndTime.ToMinutes().ToString()));
string csvFormat = "{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}";
File.WriteAllLines(resultsFormat.FormatWith("actualSimulation.txt"), new string[] { csvFormat.FormatWith("Id", "Thread", "Minimum Start Time", "Start Time", "End Time", "Duration", "Incoming", "Outgoing") }.Concat(actualSimulation.GetSpans().Select(ps =>
csvFormat.FormatWith(
ps.Id.Value,
ps.Thread,
actualSimulation.MinimumStartTimes[ps.Id].ToMinutes(),
ps.StartTime.ToMinutes(),
ps.EndTime.ToMinutes(),
ps.Duration.ToMinutes(),
data.DataflowGraph.GetIncomingEdgesCount(ps.Id),
data.DataflowGraph.GetIncomingEdgesCount(ps.Id)))));
CreateSimulationImage(actualSimulation, resultsFormat.FormatWith("actualSimulation.png"));
Console.WriteLine("Done");
}
Console.WriteLine();
writers.WriteLine("Actual Total Build Time: {0} min", data.TotalDuration.ToMinutes());
writers.WriteLine("Actual Concurrency: {0}", data.ActualConcurrency);
if (actualSimulation != null)
{
writers.WriteLine("Simulated total build time (using actual concurrency): {0} min", actualSimulation.TotalTime.ToMinutes());
}
int nameWidth;
int count = 0;
ulong criticalPathCost = 0;
foreach (var p in data.GetSortedPips(20, true, n => data.PipTypes[n] == PipType.Process, n => data.AggregateCosts[n]))
{
List <NodeId> criticalPath = new List <NodeId>();
NodeId criticalChainNode = p.Node;
while (criticalChainNode.IsValid())
{
criticalPath.Add(criticalChainNode);
criticalChainNode = data.CriticalChain[criticalChainNode];
}
p.Priority.Max(ref criticalPathCost);
writers.WriteLine("Critical Path {0}:", count++);
writers.WriteLine("Critical Path Cost: {0} min", p.Priority.ToMinutes());
writers.WriteLine("Critical Path Length: {0}", criticalPath.Count);
nameWidth = criticalPath.Select(n => data.GetName(n).Length).Max();
writers.WriteLine("Critical Path:\n {0}", string.Join(Environment.NewLine + " -> ", criticalPath
//.Where(n => data.Durations[n] > 0)
.Select(n => string.Format("{0} ({1} min) [{2}] <{3}>", data.GetName(n).PadLeft(nameWidth), data.Durations[n].ToMinutes(), data.PipIds[n], data.PipTypes[n]))));
}
nameWidth = data.LongestRunningPips.Select(n => data.GetName(n.Node).Length).Max();
writers.WriteLine();
writers.WriteLine("Top 20 Long Running Pips:");
foreach (var p in data.LongestRunningPips)
{
writers.WriteLine("{0} [{2}]: {1} min", data.GetName(p.Node).PadLeft(nameWidth), p.Priority.ToMinutes(), data.PipIds[p.Node]);
}
writers.WriteLine();
writers.WriteLine("Bottom 20 Shortest Running Processes:");
foreach (var p in data.ShortestRunningProcesses)
{
writers.WriteLine("{0} [{2}]: {1} min", data.GetName(p.Node).PadLeft(nameWidth), p.Priority.ToMinutes(), data.PipIds[p.Node]);
}
int simulationCount = 1;
int increment = 70;
int?[] threadCounts = new int?[simulationCount];
threadCounts[0] = 60;
//threadCounts[0] = 60;
//threadCounts[4] = 330;
//threadCounts[5] = 400;
//threadCounts[6] = 470;
SimulationResult[] results = new SimulationResult[simulationCount];
Parallel.For(0, simulationCount, i =>
{
var threadCount = threadCounts[i] ?? ((i + 1) * increment);
Console.WriteLine("Simulating {0}...", threadCount);
SimulationResult result = new SimulationResult(data, data.AggregateCosts);
result.Simulate((uint)threadCount);
results[i] = result;
Console.WriteLine("Done {0}", threadCount);
//Console.WriteLine("Simulating adjusted{0}...", threadCount);
//SimulationResult adjusted = new SimulationResult(data, data.ComputeAggregateCosts(result.GetAdjustedDurations(criticalPathCost)));
//result.Simulate((uint)threadCount);
//results[i + simulationCount] = result;
//Console.WriteLine("Done adjusted {0}", threadCount);
//i = i + (results.Length / 2);
//Console.WriteLine("Simulating {0}...", i);
//SimulationResult adjusted = new SimulationResult(data, result.EndTimes);
//adjusted.Simulate((uint)result.Threads.Count);
//results[i] = adjusted;
//Console.WriteLine("Done {0}", i);
});
results = results.OrderBy(s => s.Threads.Count).ToArray();
string format = "{0}, {1}, {5}, {6}";
writers.WriteLine();
writers.WriteLine("Results");
writers.WriteLine(format,
"Thread Count", // 0
"Total Execution Time (min)", // 1
"Critical Path Length", // 2
"Critical Path Tail Pip", // 3
"Critical Path Idle Time (min)", // 4
"Average Utilization", // 5
"Max Path Length", // 6
"Executed Pips" // 7
);
for (int i = 0; i < results.Length; i++)
{
var result = results[i];
writers.WriteLine(format,
result.Threads.Count,
result.TotalTime.ToMinutes(),
result.CriticalPath.Executions.Count,
result.CriticalPath.Executions.Last().Id.Value,
result.CriticalPath.IdleTime.ToMinutes(),
Math.Round((double)result.TotalActiveTime / (double)((ulong)result.Threads.Count * result.TotalTime), 3),
result.Threads.Select(t => t.Executions.Count).Max(),
result.Threads.Select(t => t.Executions.Count).Sum());
CreateSimulationImage(result, resultsFormat.FormatWith("simulation.{0}.png".FormatWith(result.Threads.Count)));
}
}
}
