/// <summary> /// Launch a new process. /// </summary> /// <param name="processInfo">application to execute</param> /// <param name="jittedMethods">Jitted method collector</param> /// <param name="processIndex">Numeric index used to prefix messages pertaining to this process in the console output</param> /// <param name="processCount">Total number of processes being executed (used for displaying progress)</param> /// <param name="progressIndex">Number of processes that have already finished (for displaying progress)</param> public void Launch(ProcessInfo processInfo, ReadyToRunJittedMethods jittedMethods, int processIndex, int processCount, int progressIndex) { Debug.Assert(_processRunner == null); Console.WriteLine($"{processIndex} / {processCount} ({(progressIndex * 100 / processCount)}%): launching: {processInfo.ProcessPath} {processInfo.Arguments}"); _processRunner = new ProcessRunner(processInfo, processIndex, processCount, jittedMethods, _processExitEvent); }
private static void BuildEtwProcesses(IEnumerable <ProcessInfo> processesToRun, int degreeOfParallelism) { using (TraceEventSession traceEventSession = new TraceEventSession("ReadyToRunTestSession")) { traceEventSession.EnableProvider(ClrTraceEventParser.ProviderGuid, TraceEventLevel.Verbose, (ulong)(ClrTraceEventParser.Keywords.Jit | ClrTraceEventParser.Keywords.Loader)); using (ReadyToRunJittedMethods jittedMethods = new ReadyToRunJittedMethods(traceEventSession, processesToRun)) { Task.Run(() => { BuildProjects(processesToRun, jittedMethods, degreeOfParallelism); traceEventSession.Stop(); }); } traceEventSession.Source.Process(); } // Append jitted method info to the logs foreach (ProcessInfo processInfo in processesToRun) { if (processInfo.CollectJittedMethods) { using (StreamWriter logWriter = new StreamWriter(processInfo.LogPath, append: true)) { logWriter.WriteLine($"Jitted methods ({processInfo.JittedMethods.Sum(moduleMethodsKvp => moduleMethodsKvp.Value.Count)} total):"); foreach (KeyValuePair <string, HashSet <string> > jittedMethodsPerModule in processInfo.JittedMethods) { foreach (string method in jittedMethodsPerModule.Value) { logWriter.WriteLine(jittedMethodsPerModule.Key + " -> " + method); } } } } } }
public ProcessRunner(ProcessInfo processInfo, int processIndex, ReadyToRunJittedMethods jittedMethods, AutoResetEvent processExitEvent) { _processInfo = processInfo; _processIndex = processIndex; _jittedMethods = jittedMethods; _processExitEvent = processExitEvent; _cancellationTokenSource = new CancellationTokenSource(); _stopwatch = new Stopwatch(); _stopwatch.Start(); _state = StateIdle; _logWriter = new StreamWriter(_processInfo.LogPath); if (_processInfo.ProcessPath.Contains(' ')) { _logWriter.Write($"\"{_processInfo.ProcessPath}\""); } else { _logWriter.Write(_processInfo.ProcessPath); } _logWriter.Write(' '); _logWriter.WriteLine(_processInfo.Arguments); _logWriter.WriteLine("<<<<"); ProcessStartInfo psi = new ProcessStartInfo() { FileName = _processInfo.ProcessPath, Arguments = _processInfo.Arguments, UseShellExecute = _processInfo.UseShellExecute, RedirectStandardOutput = true, RedirectStandardError = true, }; _process = new Process(); _process.StartInfo = psi; _process.EnableRaisingEvents = true; _process.Exited += new EventHandler(ExitEventHandler); Interlocked.Exchange(ref _state, StateRunning); _process.Start(); if (_processInfo.CollectJittedMethods) { _jittedMethods.SetProcessId(_processInfo, _process.Id); } _process.OutputDataReceived += new DataReceivedEventHandler(StandardOutputEventHandler); _process.BeginOutputReadLine(); _process.ErrorDataReceived += new DataReceivedEventHandler(StandardErrorEventHandler); _process.BeginErrorReadLine(); Task.Run(TimeoutWatchdog); }
private static void BuildProjects(IEnumerable <ProcessInfo> processesToRun, ReadyToRunJittedMethods jittedMethods, int degreeOfParallelism) { using (AutoResetEvent processExitEvent = new AutoResetEvent(initialState: false)) { ProcessSlot[] processSlots = new ProcessSlot[degreeOfParallelism]; for (int index = 0; index < degreeOfParallelism; index++) { processSlots[index] = new ProcessSlot(index, processExitEvent); } int processIndex = 0; foreach (ProcessInfo processInfo in processesToRun) { // Allocate a process slot, potentially waiting on the exit event // when all slots are busy (running) ProcessSlot freeSlot = null; do { foreach (ProcessSlot slot in processSlots) { if (slot.IsAvailable()) { freeSlot = slot; break; } } if (freeSlot == null) { // All slots are busy - wait for a process to finish processExitEvent.WaitOne(); } }while (freeSlot == null); freeSlot.Launch(processInfo, jittedMethods, ++processIndex); } // We have launched all the commands, now wait for all processes to finish bool activeProcessesExist; do { activeProcessesExist = false; foreach (ProcessSlot slot in processSlots) { if (!slot.IsAvailable()) { activeProcessesExist = true; } } if (activeProcessesExist) { processExitEvent.WaitOne(); } }while (activeProcessesExist); } }
private static void BuildEtwProcesses(int startIndex, int endIndex, int totalCount, List <ProcessInfo> processList, int degreeOfParallelism) { using (TraceEventSession traceEventSession = new TraceEventSession("ReadyToRunTestSession")) { traceEventSession.EnableProvider(ClrTraceEventParser.ProviderGuid, TraceEventLevel.Verbose, (ulong)(ClrTraceEventParser.Keywords.Jit | ClrTraceEventParser.Keywords.Loader)); using (ReadyToRunJittedMethods jittedMethods = new ReadyToRunJittedMethods(traceEventSession, processList, startIndex, endIndex)) { Task.Run(() => { BuildProjects(startIndex, endIndex, totalCount, processList, jittedMethods, degreeOfParallelism); traceEventSession.Stop(); }); } traceEventSession.Source.Process(); } // Append jitted method info to the logs for (int index = startIndex; index < endIndex; index++) { ProcessInfo processInfo = processList[index]; if (processInfo.Parameters.CollectJittedMethods) { using (StreamWriter processLogWriter = new StreamWriter(processInfo.Parameters.LogPath, append: true)) { if (processInfo.JittedMethods != null) { processLogWriter.WriteLine($"Jitted methods ({processInfo.JittedMethods.Sum(moduleMethodsKvp => moduleMethodsKvp.Value.Count)} total):"); foreach (KeyValuePair <string, HashSet <string> > jittedMethodsPerModule in processInfo.JittedMethods) { foreach (string method in jittedMethodsPerModule.Value) { processLogWriter.WriteLine(jittedMethodsPerModule.Key + " -> " + method); } } } else { processLogWriter.WriteLine("Jitted method info not available"); } } } } }
public ProcessRunner(ProcessInfo processInfo, int processIndex, int processCount, ReadyToRunJittedMethods jittedMethods, AutoResetEvent processExitEvent) { _processInfo = processInfo; _processIndex = processIndex; _processCount = processCount; _jittedMethods = jittedMethods; _processExitEvent = processExitEvent; _cancellationTokenSource = new CancellationTokenSource(); _stopwatch = new Stopwatch(); _stopwatch.Start(); _state = StateIdle; _logWriter = new StreamWriter(_processInfo.Parameters.LogPath); if (_processInfo.Parameters.ProcessPath.IndexOf(' ') >= 0) { _logWriter.Write($"\"{_processInfo.Parameters.ProcessPath}\""); } else { _logWriter.Write(_processInfo.Parameters.ProcessPath); } _logWriter.Write(' '); _logWriter.WriteLine(_processInfo.Parameters.Arguments); _logWriter.WriteLine("<<<<"); ProcessStartInfo psi = new ProcessStartInfo() { FileName = _processInfo.Parameters.ProcessPath, Arguments = _processInfo.Parameters.Arguments, UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true, }; foreach (KeyValuePair <string, string> environmentOverride in _processInfo.Parameters.EnvironmentOverrides) { psi.EnvironmentVariables[environmentOverride.Key] = environmentOverride.Value; } _process = new Process(); _process.StartInfo = psi; _process.EnableRaisingEvents = true; _process.Exited += new EventHandler(ExitEventHandler); Interlocked.Exchange(ref _state, StateRunning); _process.Start(); if (_processInfo.Parameters.CollectJittedMethods) { _jittedMethods.AddProcessMapping(_processInfo, _process); } _outputHandler = new DataReceivedEventHandler(StandardOutputEventHandler); _process.OutputDataReceived += _outputHandler; _process.BeginOutputReadLine(); _errorHandler = new DataReceivedEventHandler(StandardErrorEventHandler); _process.ErrorDataReceived += _errorHandler; _process.BeginErrorReadLine(); Task.Run(TimeoutWatchdog); }
private static void BuildEtwProcesses(int startIndex, int endIndex, int totalCount, int failureCount, List <ProcessInfo> processList, int degreeOfParallelism, bool measurePerf) { using (TraceEventSession traceEventSession = new TraceEventSession("ReadyToRunTestSession")) { traceEventSession.EnableProvider(ClrTraceEventParser.ProviderGuid, TraceEventLevel.Verbose, (ulong)(ClrTraceEventParser.Keywords.Jit | ClrTraceEventParser.Keywords.Loader)); int warmupRuns = 0; int realRuns = 1; PerfEventSourceListener perfMeasurer = null; if (measurePerf) { Debug.Assert(processList.Count == 1); warmupRuns = 1; realRuns = 5; perfMeasurer = new PerfEventSourceListener(traceEventSession, warmupRuns, realRuns); } using (ReadyToRunJittedMethods jittedMethods = new ReadyToRunJittedMethods(traceEventSession, processList, startIndex, endIndex)) { Task.Run(() => { // Warmup runs if (measurePerf) { Console.WriteLine("Warmup runs:"); for (int run = 0; run < warmupRuns; ++run) { BuildProjects(startIndex, endIndex, totalCount, failureCount, processList, jittedMethods, degreeOfParallelism); } // Wait for all the warmup events to come in before starting the real run so there is no interference perfMeasurer.WaitForWarmupFinished(); Console.WriteLine("Real runs:"); } for (int run = 0; run < realRuns; ++run) { BuildProjects(startIndex, endIndex, totalCount, failureCount, processList, jittedMethods, degreeOfParallelism); } if (measurePerf) { perfMeasurer.PrintPerfResults(); } traceEventSession.Stop(); }); } traceEventSession.Source.Process(); } // Append jitted method info to the logs for (int index = startIndex; index < endIndex; index++) { ProcessInfo processInfo = processList[index]; if (processInfo.Parameters.CollectJittedMethods) { using (StreamWriter processLogWriter = new StreamWriter(processInfo.Parameters.LogPath, append: true)) { if (processInfo.JittedMethods != null) { processLogWriter.WriteLine($"Jitted methods ({processInfo.JittedMethods.Sum(moduleMethodsKvp => moduleMethodsKvp.Value.Count)} total):"); foreach (KeyValuePair <string, HashSet <string> > jittedMethodsPerModule in processInfo.JittedMethods) { foreach (string method in jittedMethodsPerModule.Value) { processLogWriter.WriteLine(jittedMethodsPerModule.Key + " -> " + method); } } } else { processLogWriter.WriteLine("Jitted method info not available"); } } } } }
/// <summary> /// Execute a given set of mutually independent build commands with given degree of /// parallelism. /// </summary> /// <param name="processesToRun">Processes to execute in parallel</param> /// <param name="degreeOfParallelism">Maximum number of processes to execute in parallel</param> public static void Run(IEnumerable <ProcessInfo> processesToRun, int degreeOfParallelism) { int processCount = processesToRun.Count(); if (processCount < degreeOfParallelism) { // We never need a higher DOP than the number of process to execute degreeOfParallelism = processCount; } bool collectEtwTraces = processesToRun.Any(processInfo => processInfo.CollectJittedMethods); if (collectEtwTraces) { using (TraceEventSession traceEventSession = new TraceEventSession("ReadyToRunTestSession")) { traceEventSession.EnableProvider(ClrTraceEventParser.ProviderGuid, TraceEventLevel.Verbose, (ulong)(ClrTraceEventParser.Keywords.Jit | ClrTraceEventParser.Keywords.Loader)); ReadyToRunJittedMethods jittedMethods = new ReadyToRunJittedMethods(traceEventSession, processesToRun); Task.Run(() => { BuildProjects(processesToRun, jittedMethods, degreeOfParallelism); traceEventSession.Stop(); }); traceEventSession.Source.Process(); } // Append jitted method info to the logs foreach (ProcessInfo processInfo in processesToRun) { if (processInfo.CollectJittedMethods) { using (StreamWriter logWriter = new StreamWriter(processInfo.LogPath, append: true)) { logWriter.WriteLine($"Jitted methods ({processInfo.JittedMethods.Count} total):"); foreach (KeyValuePair <string, HashSet <string> > jittedMethodAndModules in processInfo.JittedMethods) { logWriter.Write(jittedMethodAndModules.Key); logWriter.Write(": "); bool first = true; foreach (string module in jittedMethodAndModules.Value) { if (first) { first = false; } else { logWriter.Write(", "); } logWriter.Write(module); } logWriter.WriteLine(); } } } } } else { BuildProjects(processesToRun, null, degreeOfParallelism); } }