public DisasmEngine(string executable, Config config, string outputPath, List<AssemblyInfo> assemblyInfoList) { _config = config; _executablePath = executable; _rootPath = outputPath; _platformPaths = config.PlatformPaths; _assemblyInfoList = assemblyInfoList; this.doGCDump = config.DumpGCInfo; this.verbose = config.DoVerboseOutput; }
public static List<AssemblyInfo> GenerateAssemblyWorklist(Config config) { bool verbose = config.DoVerboseOutput; List<string> assemblyList = new List<string>(); List<AssemblyInfo> assemblyInfoList = new List<AssemblyInfo>(); if (config.UseFileName) { assemblyList = new List<string>(); string inputFile = config.FileName; // Open file, read assemblies one per line, and add them to the assembly list. using (var inputStream = System.IO.File.Open(inputFile, FileMode.Open)) { using (var inputStreamReader = new StreamReader(inputStream)) { string line; while ((line = inputStreamReader.ReadLine()) != null) { // Each line is a path to an assembly. if (!File.Exists(line)) { Console.WriteLine("Can't find {0} skipping...", line); continue; } assemblyList.Add(line); } } } } if (config.HasUserAssemblies) { // Append command line assemblies assemblyList.AddRange(config.AssemblyList); } // Process worklist and produce the info needed for the disasm engines. foreach (var path in assemblyList) { FileAttributes attr; if (File.Exists(path) || Directory.Exists(path)) { attr = File.GetAttributes(path); } else { Console.WriteLine("Can't find assembly or directory at {0}", path); continue; } if ((attr & FileAttributes.Directory) == FileAttributes.Directory) { if (verbose) { Console.WriteLine("Processing directory: {0}", path); } // For the directory case create a stack and recursively find any // assemblies for compilation. List<AssemblyInfo> directoryAssemblyInfoList = IdentifyAssemblies(path, config); // Add info generated at this directory assemblyInfoList.AddRange(directoryAssemblyInfoList); } else { // This is the file case. AssemblyInfo info = new AssemblyInfo { Name = Path.GetFileName(path), Path = Path.GetDirectoryName(path), OutputPath = "" }; assemblyInfoList.Add(info); } } return assemblyInfoList; }
// Recursivly search for assemblies from a root path. private static List<AssemblyInfo> IdentifyAssemblies(string rootPath, Config config) { List<AssemblyInfo> assemblyInfoList = new List<AssemblyInfo>(); string fullRootPath = Path.GetFullPath(rootPath); SearchOption searchOption = (config.Recursive) ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; // Get files that could be assemblies, but discard currently // ngen'd assemblies. var subFiles = Directory.EnumerateFiles(rootPath, "*", searchOption) .Where(s => (s.EndsWith(".exe") || s.EndsWith(".dll")) && !s.Contains(".ni.")); foreach (var filePath in subFiles) { if (config.DoVerboseOutput) { Console.WriteLine("Scaning: {0}", filePath); } // skip if not an assembly if (!IsAssembly(filePath)) { continue; } string fileName = Path.GetFileName(filePath); string directoryName = Path.GetDirectoryName(filePath); string fullDirectoryName = Path.GetFullPath(directoryName); string outputPath = fullDirectoryName.Substring(fullRootPath.Length).TrimStart(Path.DirectorySeparatorChar); AssemblyInfo info = new AssemblyInfo { Name = fileName, Path = directoryName, OutputPath = outputPath }; assemblyInfoList.Add(info); } return assemblyInfoList; }
public static int DiffCommand(Config config) { string diffString = "System.Private.CoreLib.dll"; if (config.DoFrameworks) { diffString += ", framework assemblies"; } if (config.DoTestTree) { diffString += ", " + config.TestRoot; } Console.WriteLine("Beginning diff of {0}!", diffString); // Add each framework assembly to commandArgs // Create subjob that runs mcgdiff, which should be in path, with the // relevent coreclr assemblies/paths. string frameworkArgs = String.Join(" ", s_frameworkAssemblies); string testArgs = String.Join(" ", s_testDirectories); List<string> commandArgs = new List<string>(); // Set up CoreRoot commandArgs.Add("--platform"); commandArgs.Add(config.CoreRoot); commandArgs.Add("--output"); commandArgs.Add(config.OutputPath); if (config.HasBaseExeutable) { commandArgs.Add("--base"); commandArgs.Add(config.BaseExecutable); } if (config.HasDiffExecutable) { commandArgs.Add("--diff"); commandArgs.Add(config.DiffExecutable); } if (config.DoTestTree) { commandArgs.Add("--recursive"); } if (config.Verbose) { commandArgs.Add("--verbose"); } if (config.CoreLibOnly) { string coreRoot = config.CoreRoot; string fullPathAssembly = Path.Combine(coreRoot, "System.Private.CoreLib.dll"); commandArgs.Add(fullPathAssembly); } else { // Set up full framework paths foreach (var assembly in s_frameworkAssemblies) { string coreRoot = config.CoreRoot; string fullPathAssembly = Path.Combine(coreRoot, assembly); if (!File.Exists(fullPathAssembly)) { Console.WriteLine("can't find framework assembly {0}", fullPathAssembly); continue; } commandArgs.Add(fullPathAssembly); } if (config.TestRoot != null) { foreach (var dir in s_testDirectories) { string testRoot = config.TestRoot; string fullPathDir = Path.Combine(testRoot, dir); if (!Directory.Exists(fullPathDir)) { Console.WriteLine("can't find test directory {0}", fullPathDir); continue; } commandArgs.Add(fullPathDir); } } } Console.WriteLine("Diff command: {0} {1}", s_asmTool, String.Join(" ", commandArgs)); Command diffCmd = Command.Create( s_asmTool, commandArgs); // Wireup stdout/stderr so we can see outout. diffCmd.ForwardStdOut(); diffCmd.ForwardStdErr(); CommandResult result = diffCmd.Execute(); if (result.ExitCode != 0) { Console.WriteLine("Dasm command returned with {0} failures", result.ExitCode); return result.ExitCode; } // Analyze completed run. if (config.DoAnalyze == true) { List<string> analysisArgs = new List<string>(); analysisArgs.Add("--base"); analysisArgs.Add(Path.Combine(config.OutputPath, "base")); analysisArgs.Add("--diff"); analysisArgs.Add(Path.Combine(config.OutputPath, "diff")); analysisArgs.Add("--recursive"); Console.WriteLine("Analyze command: {0} {1}", s_analysisTool, String.Join(" ", analysisArgs)); Command analyzeCmd = Command.Create(s_analysisTool, analysisArgs); // Wireup stdout/stderr so we can see outout. analyzeCmd.ForwardStdOut(); analyzeCmd.ForwardStdErr(); CommandResult analyzeResult = analyzeCmd.Execute(); } return result.ExitCode; }
public static int Main(string[] args) { // Error count will be returned. Start at 0 - this will be incremented // based on the error counts derived from the DisasmEngine executions. int errorCount = 0; // Parse and store comand line options. var config = new Config(args); // Stop to attach a debugger if desired. if (config.WaitForDebugger) { WaitForDebugger(); } // Builds assemblyInfoList on mcgdiff List<AssemblyInfo> assemblyWorkList = GenerateAssemblyWorklist(config); // The disasm engine encapsulates a particular set of diffs. An engine is // produced with a given code generator and assembly list, which then produces // a set of disasm outputs if (config.GenerateBaseline) { string taggedPath = null; if (config.DoFileOutput) { string tag = "base"; if (config.HasTag) { tag = config.Tag; } taggedPath = Path.Combine(config.RootPath, tag); } DisasmEngine baseDisasm = new DisasmEngine(config.BaseExecutable, config, taggedPath, assemblyWorkList); baseDisasm.GenerateAsm(); if (baseDisasm.ErrorCount > 0) { Console.WriteLine("{0} errors compiling base set.", baseDisasm.ErrorCount); errorCount += baseDisasm.ErrorCount; } } if (config.GenerateDiff) { string taggedPath = null; if (config.DoFileOutput) { string tag = "diff"; if (config.HasTag) { tag = config.Tag; } taggedPath = Path.Combine(config.RootPath, tag); } DisasmEngine diffDisasm = new DisasmEngine(config.DiffExecutable, config, taggedPath, assemblyWorkList); diffDisasm.GenerateAsm(); if (diffDisasm.ErrorCount > 0) { Console.WriteLine("{0} errors compiling diff set.", diffDisasm.ErrorCount); errorCount += diffDisasm.ErrorCount; } } return errorCount; }
// Download zip file. It's arguable that this should be in the private static async Task DownloadZip(CIClient cic, Config config, string outputPath) { // Copy product tools to output location. bool success = cic.DownloadProduct(config, outputPath).Result; if (config.DoUnzip) { // unzip archive in place. var zipPath = Path.Combine(outputPath, "Product.zip"); ZipFile.ExtractToDirectory(zipPath, outputPath); } }
public static int InstallCommand(Config config) { var asmDiffPath = Path.Combine(config.JitDasmRoot, "asmdiff.json"); string configJson = File.ReadAllText(asmDiffPath); string tag = String.Format("{0}-{1}", config.JobName, config.Number); string toolPath = Path.Combine(config.JitDasmRoot, "tools", tag); var jObj = JObject.Parse(configJson); var tools = (JArray)jObj["tools"]; int ret = 0; // Early out if the tool is already installed. if (tools.Where(x => (string)x["tag"] == tag).Any()) { Console.WriteLine("{0} is already installed in the asmdiff.json. Remove before re-install.", tag); return -1; } // Issue cijobs command to download bits List<string> cijobsArgs = new List<string>(); cijobsArgs.Add("copy"); // Set up job name cijobsArgs.Add("--job"); cijobsArgs.Add(config.JobName); if (config.BranchName != null) { cijobsArgs.Add("--branch"); cijobsArgs.Add(config.BranchName); } if (config.DoLastSucessful) { cijobsArgs.Add("--last_successful"); } else { // Set up job number cijobsArgs.Add("--number"); cijobsArgs.Add(config.Number); } cijobsArgs.Add("--unzip"); cijobsArgs.Add("--output"); cijobsArgs.Add(toolPath); if (config.Verbose) { Console.WriteLine("ci command: {0} {1}", "cijobs", String.Join(" ", cijobsArgs)); } Command cijobsCmd = Command.Create("cijobs", cijobsArgs); // Wireup stdout/stderr so we can see outout. cijobsCmd.ForwardStdOut(); cijobsCmd.ForwardStdErr(); CommandResult result = cijobsCmd.Execute(); if (result.ExitCode != 0) { Console.WriteLine("cijobs command returned with {0} failures", result.ExitCode); return result.ExitCode; } JObject newTool = new JObject(); newTool.Add("tag", tag); // Derive underlying tool directory based on current RID. string[] platStrings = config.RID.Split('.'); string platformPath = Path.Combine(toolPath, "Product"); foreach (var dir in Directory.EnumerateDirectories(platformPath)) { if (Path.GetFileName(dir).ToUpper().Contains(platStrings[0].ToUpper())) { newTool.Add("path", Path.GetFullPath(dir)); tools.Last.AddAfterSelf(newTool); break; } } // Overwrite current asmdiff.json with new data. using (var file = File.CreateText(asmDiffPath)) { using (JsonTextWriter writer = new JsonTextWriter(file)) { writer.Formatting = Formatting.Indented; jObj.WriteTo(writer); } } return ret; }
// Based on the config, copy down the artifacts for the referenced job. // Today this is just the product bits. This code also knows how to install // the bits into the asmdiff.json config file. public static async Task Copy(CIClient cic, Config config) { if (config.LastSuccessful) { // Querry last successful build and extract the number. var builds = cic.GetJobBuilds("dotnet_coreclr", "master", config.JobName, true); if (!builds.Result.Any()) { Console.WriteLine("Last successful not found on server."); return; } Build lastSuccess = builds.Result.First(); config.Number = lastSuccess.number; } string tag = String.Format("{0}-{1}", config.JobName, config.Number); string outputPath = config.OutputPath; // Create directory if it doesn't exist. Directory.CreateDirectory(outputPath); // Pull down the zip file. DownloadZip(cic, config, outputPath).Wait(); }
// List jobs and their details from the dotnet_coreclr project on .NETCI Jenkins instance. // List functionality: // if --job is not specified, ListOption.Jobs, list jobs under branch. // (default is "master" set in Config). // if --job is specified, ListOption.Builds, list job builds by id with details. // if --job and --id is specified, ListOption.Number, list particular job instance, // status, and artifacts. // public static async Task List(CIClient cic, Config config) { switch (config.DoListOption) { case ListOption.Jobs: { var jobs = cic.GetProductJobs("dotnet_coreclr", "master").Result; if (config.MatchPattern != null) { var pattern = new Regex(config.MatchPattern); PrettyJobs(jobs.Where(x => pattern.IsMatch(x.name))); } else { PrettyJobs(jobs); } } break; case ListOption.Builds: { var builds = cic.GetJobBuilds("dotnet_coreclr", "master", config.JobName, config.LastSuccessful); if (config.LastSuccessful && builds.Result.Any()) { Console.WriteLine("Last successful build:"); } PrettyBuilds(builds.Result, config.Artifacts); } break; case ListOption.Number: { var info = cic.GetJobBuildInfo(config.JobName, config.Number); // Pretty build info PrettyBuildInfo(info.Result, config.Artifacts); } break; default: { Console.WriteLine("Unknown list option!"); } break; } }
public async Task<bool> DownloadProduct(Config config, string outputPath) { string messageString = String.Format("job/dotnet_coreclr/job/{0}/job/{1}/{2}/artifact/bin/Product/*zip*/Product.zip", config.CoreclrBranchName, config.JobName, config.Number); Console.WriteLine("Downloading: {0}", messageString); HttpResponseMessage response = await _client.GetAsync(messageString); bool downloaded = false; if (response.IsSuccessStatusCode) { var zipPath = Path.Combine(outputPath, "Product.zip"); using (var outputStream = System.IO.File.Create(zipPath)) { Stream inputStream = await response.Content.ReadAsStreamAsync(); inputStream.CopyTo(outputStream); } downloaded = true; } else { Console.WriteLine("Zip not found!"); } return downloaded; }
public CIClient(Config config) { _client = new HttpClient(); _client.BaseAddress = new Uri("http://dotnet-ci.cloudapp.net/"); _client.DefaultRequestHeaders.Accept.Clear(); _client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); }
// Main entry point. Simply set up a httpClient to access the CI // and switch on the command to invoke underlying logic. private static int Main(string[] args) { Config config = new Config(args); int error = 0; CIClient cic = new CIClient(config); Command currentCommand = config.DoCommand; switch (currentCommand) { case Command.List: { ListCommand.List(cic, config).Wait(); break; } case Command.Copy: { CopyCommand.Copy(cic, config).Wait(); break; } default: { Console.WriteLine("super bad! why no command!"); error = 1; break; } } return error; }