/// <summary>
/// Quick sanity check to look over the set of assemblies to make sure they are valid and something was
/// specified.
/// </summary>
private static bool CheckAssemblyList(Options options)
{
var anyMissing = false;
foreach (var assemblyPath in options.Assemblies)
{
if (!File.Exists(assemblyPath))
{
ConsoleUtil.WriteLine(ConsoleColor.Red, $"The file '{assemblyPath}' does not exist, is an invalid file name, or you do not have sufficient permissions to read the specified file.");
anyMissing = true;
}
}
if (anyMissing)
{
return false;
}
if (options.Assemblies.Count == 0)
{
Console.WriteLine("No test assemblies specified.");
return false;
}
return true;
}
private static List<AssemblyInfo> GetAssemblyList(Options options)
{
var scheduler = new AssemblyScheduler(options);
var list = new List<AssemblyInfo>();
foreach (var assemblyPath in options.Assemblies.OrderByDescending(x => new FileInfo(x).Length))
{
var name = Path.GetFileName(assemblyPath);
// As a starting point we will just schedule the items we know to be a performance
// bottleneck. Can adjust as we get real data.
if (name == "Roslyn.Compilers.CSharp.Emit.UnitTests.dll" ||
name == "Roslyn.Services.Editor.UnitTests.dll" ||
name == "Roslyn.Services.Editor.UnitTests2.dll" ||
name == "Roslyn.VisualStudio.Services.UnitTests.dll" ||
name == "Roslyn.Services.Editor.CSharp.UnitTests.dll" ||
name == "Roslyn.Services.Editor.VisualBasic.UnitTests.dll")
{
list.AddRange(scheduler.Schedule(assemblyPath));
}
else
{
list.Add(scheduler.CreateAssemblyInfo(assemblyPath));
}
}
return list;
}
private static void WriteLogFile(Options options)
{
var fileName = Path.Combine(Path.GetDirectoryName(options.Assemblies.First()), "runtests.log");
using (var writer = new StreamWriter(fileName, append: false))
{
Logger.WriteTo(writer);
}
// This is deliberately being checked in on a temporary basis. Need to see how this data behaves in the commit
// queue and the easiest way is to dump to the console. Once the commit queue behavior is verified this will
// be deleted.
if (Constants.IsJenkinsRun)
{
Logger.WriteTo(Console.Out);
}
Logger.Clear();
}
private static async Task<int> RunCore(Options options, CancellationToken cancellationToken)
{
if (options.MissingAssemblies.Count > 0)
{
foreach (var assemblyPath in options.MissingAssemblies)
{
ConsoleUtil.WriteLine(ConsoleColor.Red, $"The file '{assemblyPath}' does not exist, is an invalid file name, or you do not have sufficient permissions to read the specified file.");
}
return 1;
}
var testExecutor = CreateTestExecutor(options);
var testRunner = new TestRunner(options, testExecutor);
var start = DateTime.Now;
var assemblyInfoList = GetAssemblyList(options);
Console.WriteLine($"Data Storage: {testExecutor.DataStorage.Name}");
Console.WriteLine($"Running {options.Assemblies.Count()} test assemblies in {assemblyInfoList.Count} chunks");
var result = await testRunner.RunAllAsync(assemblyInfoList, cancellationToken).ConfigureAwait(true);
var ellapsed = DateTime.Now - start;
Console.WriteLine($"Test execution time: {ellapsed}");
Logger.Finish(Path.GetDirectoryName(options.Assemblies.FirstOrDefault() ?? ""));
if (CanUseWebStorage())
{
await SendRunStats(options, testExecutor.DataStorage, ellapsed, result, assemblyInfoList.Count, cancellationToken).ConfigureAwait(true);
}
if (!result.Succeeded)
{
ConsoleUtil.WriteLine(ConsoleColor.Red, $"Test failures encountered");
return 1;
}
Console.WriteLine($"All tests passed");
return options.MissingAssemblies.Any() ? 1 : 0;
}
private static async Task<int> RunCore(Options options, CancellationToken cancellationToken)
{
if (!CheckAssemblyList(options))
{
return ExitFailure;
}
var testExecutor = CreateTestExecutor(options);
var testRunner = new TestRunner(options, testExecutor);
var start = DateTime.Now;
var assemblyInfoList = GetAssemblyList(options);
Console.WriteLine($"Data Storage: {testExecutor.DataStorage.Name}");
Console.WriteLine($"Running {options.Assemblies.Count()} test assemblies in {assemblyInfoList.Count} partitions");
var result = await testRunner.RunAllAsync(assemblyInfoList, cancellationToken).ConfigureAwait(true);
var elapsed = DateTime.Now - start;
Console.WriteLine($"Test execution time: {elapsed}");
Logger.Finish(Path.GetDirectoryName(options.Assemblies.FirstOrDefault() ?? ""));
DisplayResults(options.Display, result.TestResults);
if (CanUseWebStorage())
{
await SendRunStats(options, testExecutor.DataStorage, elapsed, result, assemblyInfoList.Count, cancellationToken).ConfigureAwait(true);
}
if (!result.Succeeded)
{
ConsoleUtil.WriteLine(ConsoleColor.Red, $"Test failures encountered");
return ExitFailure;
}
Console.WriteLine($"All tests passed");
return ExitSuccess;
}
private static async Task SendRunStats(Options options, IDataStorage dataStorage, TimeSpan ellapsed, RunAllResult result, int chunkCount, CancellationToken cancellationToken)
{
var obj = new JObject();
obj["Cache"] = dataStorage.Name;
obj["EllapsedSeconds"] = (int)ellapsed.TotalSeconds;
obj["IsJenkins"] = Constants.IsJenkinsRun;
obj["Is32Bit"] = !options.Test64;
obj["AssemblyCount"] = options.Assemblies.Count;
obj["CacheCount"] = result.CacheCount;
obj["ChunkCount"] = chunkCount;
obj["Succeeded"] = result.Succeeded;
var request = new RestRequest("api/testrun", Method.POST);
request.RequestFormat = DataFormat.Json;
request.AddParameter("text/json", obj.ToString(), ParameterType.RequestBody);
try
{
var client = new RestClient(Constants.DashboardUriString);
var response = await client.ExecuteTaskAsync(request);
if (response.StatusCode != System.Net.HttpStatusCode.NoContent)
{
Logger.Log($"Unable to send results: {response.ErrorMessage}");
}
}
catch
{
Logger.Log("Unable to send results");
}
}
private static ITestExecutor CreateTestExecutor(Options options)
{
var processTestExecutor = new ProcessTestExecutor(options);
if (!options.UseCachedResults)
{
return processTestExecutor;
}
// The web caching layer is still being worked on. For now want to limit it to Roslyn developers
// and Jenkins runs by default until we work on this a bit more. Anyone reading this who wants
// to try it out should feel free to opt into this.
IDataStorage dataStorage = new LocalDataStorage();
if (CanUseWebStorage())
{
dataStorage = new WebDataStorage();
}
return new CachingTestExecutor(options, processTestExecutor, dataStorage);
}
internal static Options Parse(string[] args)
{
if (args == null || args.Any(a => a == null) || args.Length < 2)
{
return null;
}
var opt = new Options { XunitPath = args[0], UseHtml = true, UseCachedResults = true };
var index = 1;
var allGood = true;
var comp = StringComparer.OrdinalIgnoreCase;
while (index < args.Length)
{
const string optionTrait = "-trait:";
const string optionNoTrait = "-notrait:";
const string optionDisplay = "-display:";
var current = args[index];
if (comp.Equals(current, "-test64"))
{
opt.Test64 = true;
index++;
}
else if (comp.Equals(current, "-xml"))
{
opt.UseHtml = false;
index++;
}
else if (comp.Equals(current, "-nocache"))
{
opt.UseCachedResults = false;
index++;
}
else if (current.StartsWith(optionDisplay, StringComparison.OrdinalIgnoreCase))
{
Display display;
var arg = current.Substring(optionDisplay.Length);
if (Enum.TryParse(arg, ignoreCase: true, result: out display))
{
opt.Display = display;
}
else
{
Console.WriteLine($"{arg} is not a valid option for display");
allGood = false;
}
index++;
}
else if (current.StartsWith(optionTrait, StringComparison.OrdinalIgnoreCase))
{
opt.Trait = current.Substring(optionTrait.Length);
index++;
}
else if (current.StartsWith(optionNoTrait, StringComparison.OrdinalIgnoreCase))
{
opt.NoTrait = current.Substring(optionNoTrait.Length);
index++;
}
else
{
break;
}
}
try
{
opt.XunitPath = opt.Test64
? Path.Combine(opt.XunitPath, "xunit.console.exe")
: Path.Combine(opt.XunitPath, "xunit.console.x86.exe");
}
catch (ArgumentException ex)
{
Console.WriteLine($"{opt.XunitPath} is not a valid path: {ex.Message}");
return null;
}
if (!File.Exists(opt.XunitPath))
{
Console.WriteLine($"The file '{opt.XunitPath}' does not exist.");
return null;
}
opt.Assemblies = args.Skip(index).ToList();
return allGood ? opt : null;
}
internal TestRunner(Options options, ITestExecutor testExecutor)
{
_testExecutor = testExecutor;
_options = options;
}
internal ProcessTestExecutor(Options options)
{
_options = options;
}
internal TestRunner(Options options)
{
this._options = options;
}
private static async Task SendRunStats(Options options, IDataStorage dataStorage, TimeSpan elapsed, RunAllResult result, int partitionCount, CancellationToken cancellationToken)
{
var testRunData = new TestRunData()
{
Cache = dataStorage.Name,
ElapsedSeconds = (int)elapsed.TotalSeconds,
IsJenkins = Constants.IsJenkinsRun,
Is32Bit = !options.Test64,
AssemblyCount = options.Assemblies.Count,
ChunkCount = partitionCount,
CacheCount = result.CacheCount,
Succeeded = result.Succeeded
};
var request = new RestRequest("api/testData/run", Method.POST);
request.RequestFormat = DataFormat.Json;
request.AddParameter("text/json", JsonConvert.SerializeObject(testRunData), ParameterType.RequestBody);
try
{
var client = new RestClient(Constants.DashboardUriString);
var response = await client.ExecuteTaskAsync(request);
if (response.StatusCode != System.Net.HttpStatusCode.NoContent)
{
Logger.Log($"Unable to send results: {response.ErrorMessage}");
}
}
catch
{
Logger.Log("Unable to send results");
}
}
internal TestRunner(Options options)
{
_options = options;
}
internal TestRunner(Options options, ITestExecutor testExecutor)
{
_testExecutor = testExecutor;
_options = options;
}
internal static Options Parse(string[] args)
{
if (args == null || args.Any(a => a == null) || args.Length < 2)
{
return null;
}
var opt = new Options {XunitPath = args[0], UseHtml = true};
int index = 1;
var comp = StringComparer.OrdinalIgnoreCase;
while (index < args.Length)
{
var current = args[index];
if (comp.Equals(current, "-test64"))
{
opt.Test64 = true;
index++;
}
else if (comp.Equals(current, "-xml"))
{
opt.UseHtml = false;
index++;
}
else if (current.Length > 7 && current.StartsWith("-trait:", StringComparison.OrdinalIgnoreCase))
{
opt.Trait = current.Substring(7);
index++;
}
else if (current.Length > 9 && current.StartsWith("-notrait:", StringComparison.OrdinalIgnoreCase))
{
opt.NoTrait = current.Substring(9);
index++;
}
else
{
break;
}
}
try
{
opt.XunitPath = opt.Test64
? Path.Combine(opt.XunitPath, "xunit.console.exe")
: Path.Combine(opt.XunitPath, "xunit.console.x86.exe");
}
catch (ArgumentException ex)
{
Console.WriteLine($"{opt.XunitPath} is not a valid path: {ex.Message}");
return null;
}
if (!File.Exists(opt.XunitPath))
{
Console.WriteLine($"The file '{opt.XunitPath}' does not exist.");
return null;
}
opt.Assemblies = new List<string>();
opt.MissingAssemblies = new List<string>();
var assemblyArgs = args.Skip(index).ToArray();
if (!assemblyArgs.Any())
{
Console.WriteLine("No test assemblies specified.");
return null;
}
foreach (var assemblyPath in assemblyArgs)
{
if (File.Exists(assemblyPath))
{
opt.Assemblies.Add(assemblyPath);
continue;
}
opt.MissingAssemblies.Add(assemblyPath);
}
return opt;
}
internal AssemblyScheduler(Options options, int methodLimit = DefaultMethodLimit)
{
_options = options;
_methodLimit = methodLimit;
}
internal static Options Parse(string[] args)
{
if (args == null || args.Any(a => a == null) || args.Length < 2)
{
return null;
}
var comparer = StringComparer.OrdinalIgnoreCase;
bool isOption(string argument, string optionName, out string value)
{
Debug.Assert(!string.IsNullOrEmpty(optionName) && optionName[0] == '-');
if (argument.StartsWith(optionName + ":", StringComparison.OrdinalIgnoreCase))
{
value = argument.Substring(optionName.Length + 1);
return !string.IsNullOrEmpty(value);
}
value = null;
return false;
}
var opt = new Options { XunitPath = args[0], UseHtml = true, UseCachedResults = true, OutputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "TestResults") };
var index = 1;
var allGood = true;
while (index < args.Length)
{
var current = args[index];
if (comparer.Equals(current, "-test64"))
{
opt.Test64 = true;
index++;
}
else if (comparer.Equals(current, "-testVsi"))
{
opt.TestVsi = true;
opt.UseCachedResults = false;
index++;
}
else if (comparer.Equals(current, "-xml"))
{
opt.UseHtml = false;
index++;
}
else if (comparer.Equals(current, "-nocache"))
{
opt.UseCachedResults = false;
index++;
}
else if (isOption(current, "-tfm", out string targetFrameworkMoniker))
{
opt.TargetFrameworkMoniker = targetFrameworkMoniker;
index++;
}
else if (isOption(current, "-out", out string value))
{
opt.OutputDirectory = value;
index++;
}
else if (isOption(current, "-display", out value))
{
if (Enum.TryParse(value, ignoreCase: true, result: out Display display))
{
opt.Display = display;
}
else
{
Console.WriteLine($"{value} is not a valid option for display");
allGood = false;
}
index++;
}
else if (isOption(current, "-trait", out value))
{
opt.Trait = value;
index++;
}
else if (isOption(current, "-notrait", out value))
{
opt.NoTrait = value;
index++;
}
else if (isOption(current, "-timeout", out value))
{
if (int.TryParse(value, out var minutes))
{
opt.Timeout = TimeSpan.FromMinutes(minutes);
}
else
{
Console.WriteLine($"{value} is not a valid minute value for timeout");
allGood = false;
}
index++;
}
else if (isOption(current, "-procdumpPath", out value))
{
opt.ProcDumpDirectory = value;
index++;
}
else if (comparer.Equals(current, "-useprocdump"))
{
opt.UseProcDump = false;
index++;
}
else
{
break;
}
}
try
{
opt.XunitPath = opt.Test64
? Path.Combine(opt.XunitPath, "xunit.console.exe")
: Path.Combine(opt.XunitPath, "xunit.console.x86.exe");
}
catch (ArgumentException ex)
{
Console.WriteLine($"{opt.XunitPath} is not a valid path: {ex.Message}");
return null;
}
if (!File.Exists(opt.XunitPath))
{
Console.WriteLine($"The file '{opt.XunitPath}' does not exist.");
return null;
}
if (opt.UseProcDump && string.IsNullOrEmpty(opt.ProcDumpDirectory))
{
Console.WriteLine($"The option 'useprocdump' was specified but 'procdumppath' was not provided");
return null;
}
opt.Assemblies = args.Skip(index).ToList();
return allGood ? opt : null;
}