public DisasmEngine(string executable, Config config, string outputPath,
List<AssemblyInfo> assemblyInfoList)
{
_config = config;
_executablePath = executable;
_rootPath = outputPath;
_platformPaths = config.PlatformPaths;
_assemblyInfoList = assemblyInfoList;
this.doGCDump = config.DumpGCInfo;
this.verbose = config.DoVerboseOutput;
}
public static List<AssemblyInfo> GenerateAssemblyWorklist(Config config)
{
bool verbose = config.DoVerboseOutput;
List<string> assemblyList = new List<string>();
List<AssemblyInfo> assemblyInfoList = new List<AssemblyInfo>();
if (config.UseFileName)
{
assemblyList = new List<string>();
string inputFile = config.FileName;
// Open file, read assemblies one per line, and add them to the assembly list.
using (var inputStream = System.IO.File.Open(inputFile, FileMode.Open))
{
using (var inputStreamReader = new StreamReader(inputStream))
{
string line;
while ((line = inputStreamReader.ReadLine()) != null)
{
// Each line is a path to an assembly.
if (!File.Exists(line))
{
Console.WriteLine("Can't find {0} skipping...", line);
continue;
}
assemblyList.Add(line);
}
}
}
}
if (config.HasUserAssemblies)
{
// Append command line assemblies
assemblyList.AddRange(config.AssemblyList);
}
// Process worklist and produce the info needed for the disasm engines.
foreach (var path in assemblyList)
{
FileAttributes attr;
if (File.Exists(path) || Directory.Exists(path))
{
attr = File.GetAttributes(path);
}
else
{
Console.WriteLine("Can't find assembly or directory at {0}", path);
continue;
}
if ((attr & FileAttributes.Directory) == FileAttributes.Directory)
{
if (verbose)
{
Console.WriteLine("Processing directory: {0}", path);
}
// For the directory case create a stack and recursively find any
// assemblies for compilation.
List<AssemblyInfo> directoryAssemblyInfoList = IdentifyAssemblies(path,
config);
// Add info generated at this directory
assemblyInfoList.AddRange(directoryAssemblyInfoList);
}
else
{
// This is the file case.
AssemblyInfo info = new AssemblyInfo
{
Name = Path.GetFileName(path),
Path = Path.GetDirectoryName(path),
OutputPath = ""
};
assemblyInfoList.Add(info);
}
}
return assemblyInfoList;
}
// Recursivly search for assemblies from a root path.
private static List<AssemblyInfo> IdentifyAssemblies(string rootPath, Config config)
{
List<AssemblyInfo> assemblyInfoList = new List<AssemblyInfo>();
string fullRootPath = Path.GetFullPath(rootPath);
SearchOption searchOption = (config.Recursive) ?
SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
// Get files that could be assemblies, but discard currently
// ngen'd assemblies.
var subFiles = Directory.EnumerateFiles(rootPath, "*", searchOption)
.Where(s => (s.EndsWith(".exe") || s.EndsWith(".dll"))
&& !s.Contains(".ni."));
foreach (var filePath in subFiles)
{
if (config.DoVerboseOutput)
{
Console.WriteLine("Scaning: {0}", filePath);
}
// skip if not an assembly
if (!IsAssembly(filePath))
{
continue;
}
string fileName = Path.GetFileName(filePath);
string directoryName = Path.GetDirectoryName(filePath);
string fullDirectoryName = Path.GetFullPath(directoryName);
string outputPath = fullDirectoryName.Substring(fullRootPath.Length).TrimStart(Path.DirectorySeparatorChar);
AssemblyInfo info = new AssemblyInfo
{
Name = fileName,
Path = directoryName,
OutputPath = outputPath
};
assemblyInfoList.Add(info);
}
return assemblyInfoList;
}
public static int DiffCommand(Config config)
{
string diffString = "System.Private.CoreLib.dll";
if (config.DoFrameworks)
{
diffString += ", framework assemblies";
}
if (config.DoTestTree)
{
diffString += ", " + config.TestRoot;
}
Console.WriteLine("Beginning diff of {0}!", diffString);
// Add each framework assembly to commandArgs
// Create subjob that runs mcgdiff, which should be in path, with the
// relevent coreclr assemblies/paths.
string frameworkArgs = String.Join(" ", s_frameworkAssemblies);
string testArgs = String.Join(" ", s_testDirectories);
List<string> commandArgs = new List<string>();
// Set up CoreRoot
commandArgs.Add("--platform");
commandArgs.Add(config.CoreRoot);
commandArgs.Add("--output");
commandArgs.Add(config.OutputPath);
if (config.HasBaseExeutable)
{
commandArgs.Add("--base");
commandArgs.Add(config.BaseExecutable);
}
if (config.HasDiffExecutable)
{
commandArgs.Add("--diff");
commandArgs.Add(config.DiffExecutable);
}
if (config.DoTestTree)
{
commandArgs.Add("--recursive");
}
if (config.Verbose)
{
commandArgs.Add("--verbose");
}
if (config.CoreLibOnly)
{
string coreRoot = config.CoreRoot;
string fullPathAssembly = Path.Combine(coreRoot, "System.Private.CoreLib.dll");
commandArgs.Add(fullPathAssembly);
}
else
{
// Set up full framework paths
foreach (var assembly in s_frameworkAssemblies)
{
string coreRoot = config.CoreRoot;
string fullPathAssembly = Path.Combine(coreRoot, assembly);
if (!File.Exists(fullPathAssembly))
{
Console.WriteLine("can't find framework assembly {0}", fullPathAssembly);
continue;
}
commandArgs.Add(fullPathAssembly);
}
if (config.TestRoot != null)
{
foreach (var dir in s_testDirectories)
{
string testRoot = config.TestRoot;
string fullPathDir = Path.Combine(testRoot, dir);
if (!Directory.Exists(fullPathDir))
{
Console.WriteLine("can't find test directory {0}", fullPathDir);
continue;
}
commandArgs.Add(fullPathDir);
}
}
}
Console.WriteLine("Diff command: {0} {1}", s_asmTool, String.Join(" ", commandArgs));
Command diffCmd = Command.Create(
s_asmTool,
commandArgs);
// Wireup stdout/stderr so we can see outout.
diffCmd.ForwardStdOut();
diffCmd.ForwardStdErr();
CommandResult result = diffCmd.Execute();
if (result.ExitCode != 0)
{
Console.WriteLine("Dasm command returned with {0} failures", result.ExitCode);
return result.ExitCode;
}
// Analyze completed run.
if (config.DoAnalyze == true)
{
List<string> analysisArgs = new List<string>();
analysisArgs.Add("--base");
analysisArgs.Add(Path.Combine(config.OutputPath, "base"));
analysisArgs.Add("--diff");
analysisArgs.Add(Path.Combine(config.OutputPath, "diff"));
analysisArgs.Add("--recursive");
Console.WriteLine("Analyze command: {0} {1}",
s_analysisTool, String.Join(" ", analysisArgs));
Command analyzeCmd = Command.Create(s_analysisTool, analysisArgs);
// Wireup stdout/stderr so we can see outout.
analyzeCmd.ForwardStdOut();
analyzeCmd.ForwardStdErr();
CommandResult analyzeResult = analyzeCmd.Execute();
}
return result.ExitCode;
}
public static int Main(string[] args)
{
// Error count will be returned. Start at 0 - this will be incremented
// based on the error counts derived from the DisasmEngine executions.
int errorCount = 0;
// Parse and store comand line options.
var config = new Config(args);
// Stop to attach a debugger if desired.
if (config.WaitForDebugger)
{
WaitForDebugger();
}
// Builds assemblyInfoList on mcgdiff
List<AssemblyInfo> assemblyWorkList = GenerateAssemblyWorklist(config);
// The disasm engine encapsulates a particular set of diffs. An engine is
// produced with a given code generator and assembly list, which then produces
// a set of disasm outputs
if (config.GenerateBaseline)
{
string taggedPath = null;
if (config.DoFileOutput)
{
string tag = "base";
if (config.HasTag)
{
tag = config.Tag;
}
taggedPath = Path.Combine(config.RootPath, tag);
}
DisasmEngine baseDisasm = new DisasmEngine(config.BaseExecutable, config, taggedPath, assemblyWorkList);
baseDisasm.GenerateAsm();
if (baseDisasm.ErrorCount > 0)
{
Console.WriteLine("{0} errors compiling base set.", baseDisasm.ErrorCount);
errorCount += baseDisasm.ErrorCount;
}
}
if (config.GenerateDiff)
{
string taggedPath = null;
if (config.DoFileOutput)
{
string tag = "diff";
if (config.HasTag)
{
tag = config.Tag;
}
taggedPath = Path.Combine(config.RootPath, tag);
}
DisasmEngine diffDisasm = new DisasmEngine(config.DiffExecutable, config, taggedPath, assemblyWorkList);
diffDisasm.GenerateAsm();
if (diffDisasm.ErrorCount > 0)
{
Console.WriteLine("{0} errors compiling diff set.", diffDisasm.ErrorCount);
errorCount += diffDisasm.ErrorCount;
}
}
return errorCount;
}
// Download zip file. It's arguable that this should be in the
private static async Task DownloadZip(CIClient cic, Config config, string outputPath)
{
// Copy product tools to output location.
bool success = cic.DownloadProduct(config, outputPath).Result;
if (config.DoUnzip)
{
// unzip archive in place.
var zipPath = Path.Combine(outputPath, "Product.zip");
ZipFile.ExtractToDirectory(zipPath, outputPath);
}
}
public static int InstallCommand(Config config)
{
var asmDiffPath = Path.Combine(config.JitDasmRoot, "asmdiff.json");
string configJson = File.ReadAllText(asmDiffPath);
string tag = String.Format("{0}-{1}", config.JobName, config.Number);
string toolPath = Path.Combine(config.JitDasmRoot, "tools", tag);
var jObj = JObject.Parse(configJson);
var tools = (JArray)jObj["tools"];
int ret = 0;
// Early out if the tool is already installed.
if (tools.Where(x => (string)x["tag"] == tag).Any())
{
Console.WriteLine("{0} is already installed in the asmdiff.json. Remove before re-install.", tag);
return -1;
}
// Issue cijobs command to download bits
List<string> cijobsArgs = new List<string>();
cijobsArgs.Add("copy");
// Set up job name
cijobsArgs.Add("--job");
cijobsArgs.Add(config.JobName);
if (config.BranchName != null)
{
cijobsArgs.Add("--branch");
cijobsArgs.Add(config.BranchName);
}
if (config.DoLastSucessful)
{
cijobsArgs.Add("--last_successful");
}
else
{
// Set up job number
cijobsArgs.Add("--number");
cijobsArgs.Add(config.Number);
}
cijobsArgs.Add("--unzip");
cijobsArgs.Add("--output");
cijobsArgs.Add(toolPath);
if (config.Verbose)
{
Console.WriteLine("ci command: {0} {1}", "cijobs", String.Join(" ", cijobsArgs));
}
Command cijobsCmd = Command.Create("cijobs", cijobsArgs);
// Wireup stdout/stderr so we can see outout.
cijobsCmd.ForwardStdOut();
cijobsCmd.ForwardStdErr();
CommandResult result = cijobsCmd.Execute();
if (result.ExitCode != 0)
{
Console.WriteLine("cijobs command returned with {0} failures", result.ExitCode);
return result.ExitCode;
}
JObject newTool = new JObject();
newTool.Add("tag", tag);
// Derive underlying tool directory based on current RID.
string[] platStrings = config.RID.Split('.');
string platformPath = Path.Combine(toolPath, "Product");
foreach (var dir in Directory.EnumerateDirectories(platformPath))
{
if (Path.GetFileName(dir).ToUpper().Contains(platStrings[0].ToUpper()))
{
newTool.Add("path", Path.GetFullPath(dir));
tools.Last.AddAfterSelf(newTool);
break;
}
}
// Overwrite current asmdiff.json with new data.
using (var file = File.CreateText(asmDiffPath))
{
using (JsonTextWriter writer = new JsonTextWriter(file))
{
writer.Formatting = Formatting.Indented;
jObj.WriteTo(writer);
}
}
return ret;
}
// Based on the config, copy down the artifacts for the referenced job.
// Today this is just the product bits. This code also knows how to install
// the bits into the asmdiff.json config file.
public static async Task Copy(CIClient cic, Config config)
{
if (config.LastSuccessful)
{
// Querry last successful build and extract the number.
var builds = cic.GetJobBuilds("dotnet_coreclr",
"master", config.JobName, true);
if (!builds.Result.Any())
{
Console.WriteLine("Last successful not found on server.");
return;
}
Build lastSuccess = builds.Result.First();
config.Number = lastSuccess.number;
}
string tag = String.Format("{0}-{1}", config.JobName, config.Number);
string outputPath = config.OutputPath;
// Create directory if it doesn't exist.
Directory.CreateDirectory(outputPath);
// Pull down the zip file.
DownloadZip(cic, config, outputPath).Wait();
}
// List jobs and their details from the dotnet_coreclr project on .NETCI Jenkins instance.
// List functionality:
// if --job is not specified, ListOption.Jobs, list jobs under branch.
// (default is "master" set in Config).
// if --job is specified, ListOption.Builds, list job builds by id with details.
// if --job and --id is specified, ListOption.Number, list particular job instance,
// status, and artifacts.
//
public static async Task List(CIClient cic, Config config)
{
switch (config.DoListOption)
{
case ListOption.Jobs:
{
var jobs = cic.GetProductJobs("dotnet_coreclr", "master").Result;
if (config.MatchPattern != null)
{
var pattern = new Regex(config.MatchPattern);
PrettyJobs(jobs.Where(x => pattern.IsMatch(x.name)));
}
else
{
PrettyJobs(jobs);
}
}
break;
case ListOption.Builds:
{
var builds = cic.GetJobBuilds("dotnet_coreclr",
"master", config.JobName, config.LastSuccessful);
if (config.LastSuccessful && builds.Result.Any())
{
Console.WriteLine("Last successful build:");
}
PrettyBuilds(builds.Result, config.Artifacts);
}
break;
case ListOption.Number:
{
var info = cic.GetJobBuildInfo(config.JobName, config.Number);
// Pretty build info
PrettyBuildInfo(info.Result, config.Artifacts);
}
break;
default:
{
Console.WriteLine("Unknown list option!");
}
break;
}
}
public async Task<bool> DownloadProduct(Config config, string outputPath)
{
string messageString
= String.Format("job/dotnet_coreclr/job/{0}/job/{1}/{2}/artifact/bin/Product/*zip*/Product.zip",
config.CoreclrBranchName, config.JobName, config.Number);
Console.WriteLine("Downloading: {0}", messageString);
HttpResponseMessage response = await _client.GetAsync(messageString);
bool downloaded = false;
if (response.IsSuccessStatusCode)
{
var zipPath = Path.Combine(outputPath, "Product.zip");
using (var outputStream = System.IO.File.Create(zipPath))
{
Stream inputStream = await response.Content.ReadAsStreamAsync();
inputStream.CopyTo(outputStream);
}
downloaded = true;
}
else
{
Console.WriteLine("Zip not found!");
}
return downloaded;
}
public CIClient(Config config)
{
_client = new HttpClient();
_client.BaseAddress = new Uri("http://dotnet-ci.cloudapp.net/");
_client.DefaultRequestHeaders.Accept.Clear();
_client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
}
// Main entry point. Simply set up a httpClient to access the CI
// and switch on the command to invoke underlying logic.
private static int Main(string[] args)
{
Config config = new Config(args);
int error = 0;
CIClient cic = new CIClient(config);
Command currentCommand = config.DoCommand;
switch (currentCommand)
{
case Command.List:
{
ListCommand.List(cic, config).Wait();
break;
}
case Command.Copy:
{
CopyCommand.Copy(cic, config).Wait();
break;
}
default:
{
Console.WriteLine("super bad! why no command!");
error = 1;
break;
}
}
return error;
}