public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { // Scrub the "makeflags" environment variable from our environment // so we don't pass it down to our worker process. Some users may // have things there that conflict with our usage. Process myProcess = System.Diagnostics.Process.GetCurrentProcess(); StringDictionary environmentVariables = myProcess.StartInfo.EnvironmentVariables; environmentVariables.Remove("MAKEFLAGS"); List <string> args = new List <string>(); args.Add(string.Format("OBJ={0}\\obj", workingDirectory.PathTo(this.outputPathSuffix))); args.Add(string.Format("BIN={0}", workingDirectory.PathTo(this.outputPathSuffix))); args.Add("-f"); args.Add(workingDirectory.PathTo(this.makefile)); // TODO: Remove reliance on workingDirOverride, which currently hides dependency issues and other problems. return(new ProcessInvokeAsyncWorker( workingDirectory, this, getNmakeExecutable().getRelativePath(), ////"c:/Program Files (x86)/Microsoft Visual Studio 12.0/VC/bin/nmake.exe", args.ToArray(), ProcessExitCodeHandling.NonzeroIsFailure, workingDirOverride: IronRootDirectory.PathTo(this.makefile.getDirPath()), failureBase: getDiagnosticsBase(), //allowAbsoluteExe: true, allowAbsoluteArgs: true)); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { // Scrub the "makeflags" environment variable from our environment // so we don't pass it down to our worker process. Some users may // have things there that conflict with our usage. Process myProcess = System.Diagnostics.Process.GetCurrentProcess(); StringDictionary environmentVariables = myProcess.StartInfo.EnvironmentVariables; environmentVariables.Remove("MAKEFLAGS"); List<string> args = new List<string>(); args.Add(string.Format("OBJ={0}\\obj", workingDirectory.PathTo(this.outputPathSuffix))); args.Add(string.Format("BIN={0}", workingDirectory.PathTo(this.outputPathSuffix))); args.Add("-f"); args.Add(workingDirectory.PathTo(this.makefile)); // TODO: Remove reliance on workingDirOverride, which currently hides dependency issues and other problems. return new ProcessInvokeAsyncWorker( workingDirectory, this, getNmakeExecutable().getRelativePath(), ////"c:/Program Files (x86)/Microsoft Visual Studio 12.0/VC/bin/nmake.exe", args.ToArray(), ProcessExitCodeHandling.NonzeroIsFailure, workingDirOverride: IronRootDirectory.PathTo(this.makefile.getDirPath()), failureBase: getDiagnosticsBase(), //allowAbsoluteExe: true, allowAbsoluteArgs: true); }
public static void propagatePrivateImports( WorkingDirectory workingDirectory, IContextGeneratingVerb contextVerb, BuildObject srcobj, BuildObject dstobj) { // Rewrite basm output to propagate any import statements from the beat file. // TODO this step really should be a beat function, not part of the build system. IEnumerable <BeatIncludes.LabeledInclude> beatImports = getBeatFlavoredShallowIncludesLabeled(contextVerb, srcobj); StringBuilder sb = new StringBuilder(); foreach (BeatIncludes.LabeledInclude li in beatImports) { sb.Append("//-"); sb.Append(li.importFilter == BeatIncludes.ImportFilter.ForBasmOnly ? "private-basmonly-import" : "private-import"); sb.Append(" "); sb.Append(li.buildObject.getFileNameWithoutExtension()); sb.AppendLine(";"); } // REVIEW: Improve upon this round-about way of prepending to a file? string beatOutput = File.ReadAllText(workingDirectory.PathTo(dstobj)); File.Delete(workingDirectory.PathTo(dstobj)); File.WriteAllText(workingDirectory.PathTo(dstobj), sb.ToString() + beatOutput); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { // If we were given an optional dafny input, let this build know about the resulting verb's output. // Note: this is deliberately written to break if someone changes DafnyCompileOneVerb to have multiple outputs. if (this.dafnyCompileOneVerb != null) { File.Copy( workingDirectory.PathTo(this.dafnyCompileOneVerb.getOutputs().Single()), Path.Combine(workingDirectory.PathTo(this.outputPathSuffix), "DafnyDerivedInput.cs")); } List <string> args = new List <string>(); args.Add(string.Format("/p:OutDir={0}", workingDirectory.PathTo(this.outputPathSuffix))); args.Add(string.Format("/p:Configuration={0}", this.releaseBuild ? "Release" : "Debug")); ////args.Add("/fileLogger"); // Uncomment to log MSBuild execution. args.Add(workingDirectory.PathTo(this.solutionFile)); // TODO: Fix absolute path to MSBuild.exe (at least use %SystemRoot%)! return(new ProcessInvokeAsyncWorker( workingDirectory, this, "c:\\Windows\\Microsoft.NET\\Framework\\v4.0.30319\\MSBuild.exe", args.ToArray(), ProcessExitCodeHandling.NonzeroIsFailure, failureBase: getDiagnosticsBase(), allowAbsoluteExe: true, allowAbsoluteArgs: true)); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { Disposition disposition = new Fresh(); if (this.verifyVerb != null) { VerificationResult verificationResult = VerificationResult.fromXmlFile(this.verifyVerb.getOutputs().Single()); if (!verificationResult.pass) { disposition = new Failed(); } } if (!(disposition is Failed)) { foreach (var o in this.buildVerb.getOutputs()) { if (o.getExtension() == ".exe") { File.Copy(workingDirectory.PathTo(o), workingDirectory.PathTo(this.exeOutput), overwrite: true); } else { var dest = this.RelocateBuildObjectToExeDirectory(o); File.Copy( workingDirectory.PathTo(o), workingDirectory.PathTo(dest), overwrite: true); } } } return(new VerbSyncWorker(workingDirectory, disposition)); }
/// <summary> /// Reads a build object from the local filesystem and stores it in the /// cache. /// </summary> /// <param name="workingDirectory"> /// Private directory for verb execution. /// </param> /// <param name="obj">The build object to store in the cache.</param> /// <param name="disposition"> /// Disposition of verb which created this object (if known). /// </param> /// <returns>A BuildObjectValuePointer describing the object.</returns> public BuildObjectValuePointer Store(WorkingDirectory workingDirectory, BuildObject obj, Disposition disposition) { string contentHash = Util.hashFilesystemPath(workingDirectory.PathTo(obj)); this.itemCache.StoreItemFromFile(ItemCacheContainer.Objects, contentHash, workingDirectory.PathTo(obj)); this.Add(obj, disposition, contentHash, null); return new BuildObjectValuePointer(contentHash, obj.getRelativePath()); }
// REVIEW: Make this private? internal void injectAnnotations(WorkingDirectory workingDirectory, BuildObject dest, string commentToken) { string annotations = this.emit(commentToken); // REVIEW: Improve upon this round-about way of prepending to a file? string destStr = File.ReadAllText(workingDirectory.PathTo(dest)); File.Delete(workingDirectory.PathTo(dest)); File.WriteAllText(workingDirectory.PathTo(dest), annotations + destStr); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { List <string> args = this.getArgs(); // We use workingDirOverride flag here to change the path that the // process starts in to one that is still in our workingDirectory. // So this isn't so bad. It would be better if NuBuild would only // let us supply a relative path here, however. string overrideDir = null; if (this.getWorkingDir() != null) { overrideDir = workingDirectory.PathTo(this.getWorkingDir()); } return(new ProcessInvokeAsyncWorker( workingDirectory, this, this.getSymDiffMergeExecutable().getRelativePath(), args.ToArray(), ProcessExitCodeHandling.NonzeroIsFailure, getDiagnosticsBase(), captureStdout: this.getOutputFile(), workingDirOverride: overrideDir)); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { List<string> args = this.getArgs(); preprocess(workingDirectory); // We use workingDirOverride flag here to change the path that the // process starts in to one that is still in our workingDirectory. // So this isn't so bad. It would be better if NuBuild would only // let us supply a relative path here, however. string overrideDir = null; if (this.getWorkingDir() != null) { overrideDir = workingDirectory.PathTo(this.getWorkingDir()); } return new ProcessInvokeAsyncWorker( workingDirectory, this, getSymDiffExecutable().getRelativePath(), args.ToArray(), ProcessExitCodeHandling.NonzeroIsFailure, getDiagnosticsBase(), workingDirOverride: overrideDir, returnStandardOut: true); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { if (false) { #pragma warning disable 162 File.WriteAllText(workingDirectory.PathTo(getOutputFile()), "Verification disabled temporarily for debugging"); return(new VerbSyncWorker(workingDirectory, new Fresh())); #pragma warning restore 162 } List <string> args = new List <string>(); args.Add("/noinfer"); args.Add("/typeEncoding:m"); args.Add("/z3opt:ARITH_RANDOM_SEED=1"); args.Add("/timeLimit:" + timeLimit); args.AddRange(getFlags()); args.Add(bplInput.getRelativePath()); return(new ProcessInvokeAsyncWorker( workingDirectory, this, getBoogieExecutable().getRelativePath(), args.ToArray(), ProcessExitCodeHandling.NonzeroIsOkay, getDiagnosticsBase(), allowCloudExecution: true, returnStandardOut: true, returnStandardError: true)); }
/// <summary> /// Prepares the working directory tree for a verb's execution. /// </summary> /// <param name="verb">The verb whose execution we're preparing for.</param> private void PrepareForVerb(WorkingDirectory workingDirectory, IVerb verb) { // Debugging aide: write out the abstract id for this verb. File.WriteAllText(workingDirectory.PathTo("Debug.txt"), verb.getAbstractIdentifier().ToString()); Repository repository = BuildEngine.theEngine.Repository; // Copy all verb inputs from the item cache to here. DependencyDisposition ddisp; foreach (BuildObject input in verb.getDependencies(out ddisp)) { if (!(input is VirtualBuildObject)) { workingDirectory.CreateDirectoryFor(input); // REVIEW: No longer needed? repository.Fetch(workingDirectory, input); } } // Ensures that the directory tree for each of the verb's outputs exists. foreach (BuildObject output in verb.getOutputs()) { workingDirectory.CreateDirectoryFor(output); } }
public override Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { if (!(disposition is Failed)) { File.WriteAllText(workingDirectory.PathTo(this.getOutputFile()), stdout); } return disposition; }
public override Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { if (!(disposition is Failed)) { File.WriteAllText(workingDirectory.PathTo(this.getOutputFile()), stdout); } return(disposition); }
/// <summary> /// Reads a build object from the local filesystem and stores it in the /// cache. /// </summary> /// <param name="workingDirectory"> /// Private directory for verb execution. /// </param> /// <param name="obj">The build object to store in the cache.</param> /// <param name="disposition"> /// Disposition of verb which created this object (if known). /// </param> /// <returns>A BuildObjectValuePointer describing the object.</returns> public BuildObjectValuePointer Store(WorkingDirectory workingDirectory, BuildObject obj, Disposition disposition) { string contentHash = Util.hashFilesystemPath(workingDirectory.PathTo(obj)); this.itemCache.StoreItemFromFile(ItemCacheContainer.Objects, contentHash, workingDirectory.PathTo(obj)); this.Add(obj, disposition, contentHash, null); return(new BuildObjectValuePointer(contentHash, obj.getRelativePath())); }
private void writeBootFile(WorkingDirectory workingDirectory) { List <string> lines = new List <string>(); lines.Add(this.mkBootFileEntry(workingDirectory, this.loaderCopy)); lines.Add(this.mkBootFileEntry(workingDirectory, this.appExecutableCopy)); File.WriteAllLines(workingDirectory.PathTo(this.bootIniFile), lines); }
public Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { var dis = disposition; string cSharpPath = Path.ChangeExtension(workingDirectory.PathTo(this.expandedSource), CSharpExt); if (!File.Exists(cSharpPath)) { // Dafny has a bug where compilation fails but result code is still 0. dis = new Failed(); } if (dis is Fresh) { this.RewriteCSharpFile(cSharpPath, workingDirectory.PathTo(this.output)); } return(dis); }
public Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { VerificationResult vr = new VerificationResult( bplInput.getRelativePath(), cpuTimeSeconds, stdout, stderr, new VerificationResultBoogieParser()); vr.addBasicPresentation(); vr.toXmlFile(workingDirectory.PathTo(getOutputFile())); setWasRejectableFailure(vr.wasOnlyTimeouts()); return disposition; }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { if (this.verificationRequest.isComplete()) { // If the verification succeeded, then we convert the untrusted exe into a trusted exe (via a copy). VerificationResult vr = VerificationResult.fromXmlFile(this.verifyResultsVerb.getOutputFile()); if (!vr.pass) { return(new VerbSyncWorker(workingDirectory, new Failed())); } File.Copy(workingDirectory.PathTo(this.srcObject), workingDirectory.PathTo(this.outputObject), true); // True => Overwrite } else { UnverifiedSentinelVirtualContents contents = new UnverifiedSentinelVirtualContents(); BuildEngine.theEngine.Repository.StoreVirtual(this.outputObject, new Fresh(), contents); } return(new VerbSyncWorker(workingDirectory, new Fresh())); }
public Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { VerificationResult vr = new VerificationResult( this.dfysource.getRelativePath(), cpuTimeSeconds, stdout, stderr, new VerificationResultDafnyParser()); vr.addBasicPresentation(); vr.toXmlFile(workingDirectory.PathTo(this.getOutputFile())); this.setWasRejectableFailure(!vr.pass); return(disposition); }
public virtual Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { if (!(disposition is Failed)) { // Check that the executable isn't too large. long exeSize = new FileInfo(workingDirectory.PathTo(this.outputObject)).Length; if (exeSize > this.maxExeSize) { return(new Failed("Executable too big")); } } return(disposition); }
private static IEnumerable <string> extractImportStatements(WorkingDirectory workingDirectory, BuildObject obj) { // Well, it might be nice to use BeatExtensions.propagatePrivateImports, but that requires // a context to interpret the input imports. We don't really want to cons up yet ANOTHER // intermediate context, so here's a temporary workaround. Caution; may be brittle. IEnumerable <string> imports = File.ReadAllLines(workingDirectory.PathTo(obj)) .Where(line => line.Contains("private-import")); // Note that dafny_Main_i didn't really expect us to steal its // imports, so it hasn't conditioned the Checked and Trusted imports to be beat-resistant. imports = imports.Select( line => line.Contains("Checked") || line.Contains("Trusted") ? line.Replace("private-import", "private-basmonly-import") : line); return(imports); }
/// <summary> /// Expand a dafny source file to include all of its includes inline. /// </summary> /// <param name="workingDirectory">Temporary working directory to use.</param> /// <param name="input">Source build object.</param> /// <param name="output">Where to create build object for expanded source file.</param> public void ExpandDafny(WorkingDirectory workingDirectory, SourcePath input, SourcePath output) { // Prepare the output stream. using (TextWriter outWriter = new StreamWriter(workingDirectory.PathTo(output))) { // Stash away a few things for use by our recursive helper function. this.workingDirectory = workingDirectory; this.outputWriter = outWriter; this.visited = new HashSet <string>(); // Recursively expand the initial Dafny source file to inline all of its includes. this.ExpandDafnyRecurse(input); } // Cache the output file in the Repository. BuildEngine.theEngine.Repository.Store(workingDirectory, output, new Fresh()); }
/// <summary> /// Fetches a build object and stores it in the local filesystem. /// </summary> /// <param name="workingDirectory"> /// Directory under which to store the fetched object. /// </param> /// <param name="obj">The object to fetch.</param> public void Fetch(WorkingDirectory workingDirectory, BuildObject obj) { RepositoryEntry entry = this.FetchFresh(obj); // REVIEW: best way to determine this is a source file? ItemCacheContainer container; if (obj is SourcePath) { container = ItemCacheContainer.Sources; } else { container = ItemCacheContainer.Objects; } this.itemCache.FetchItemToFile(container, entry.Hash, workingDirectory.PathTo(obj)); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { BuildObject absDestPath = this.getAbsDestPath(); ////Directory.Delete(vfs.ToPath(absDestPath), true); // This verb should be the only one writing here, so let's keep it tidy. Directory.CreateDirectory(workingDirectory.PathTo(absDestPath)); // REVIEW: Shouldn't PrepareForVerb already do this? string dafnyccExecutable = this.getExecutable().getRelativePath(); List<string> args = new List<string>(); args.AddRange(this.getRootArgs().Select<SourcePath, string>(sp => sp.getRelativePath())); args.Add("/outdir:" + this.getDestPath()); this.addExtraArgs(args); return new ProcessInvokeAsyncWorker( workingDirectory, this, dafnyccExecutable, args.ToArray(), ProcessExitCodeHandling.NonzeroIsFailure, getDiagnosticsBase()); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { BuildObject absDestPath = this.getAbsDestPath(); ////Directory.Delete(vfs.ToPath(absDestPath), true); // This verb should be the only one writing here, so let's keep it tidy. Directory.CreateDirectory(workingDirectory.PathTo(absDestPath)); // REVIEW: Shouldn't PrepareForVerb already do this? string dafnyccExecutable = this.getExecutable().getRelativePath(); List <string> args = new List <string>(); args.AddRange(this.getRootArgs().Select <SourcePath, string>(sp => sp.getRelativePath())); args.Add("/outdir:" + this.getDestPath()); this.addExtraArgs(args); return(new ProcessInvokeAsyncWorker( workingDirectory, this, dafnyccExecutable, args.ToArray(), ProcessExitCodeHandling.NonzeroIsFailure, getDiagnosticsBase())); }
public Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { if (disposition is Failed) { return(disposition); } HashSet <string> createdFiles = new HashSet <string>(Directory.GetFiles(workingDirectory.PathTo(this.getAbsDestPath())).Select(path => Path.GetFileName(path))); HashSet <string> expectedFiles = new HashSet <string>(this.getOutputs().Select(obj => obj.getFileName())); // DafnyCC/DafnySpec process a big batch of files together. Did we correctly understand what it did? if (!createdFiles.SetEquals(expectedFiles)) { // REVIEW: These are never used by anything. Remove? bool dummy = createdFiles.SetEquals(expectedFiles); int missing = expectedFiles.Except(createdFiles).Count(); int extra = createdFiles.Except(expectedFiles).Count(); string msg = "Missing files: " + string.Join(",", expectedFiles.Except(createdFiles)) + "\n" + " Extra files: " + string.Join(",", createdFiles.Except(expectedFiles)); return(new Failed(msg)); } // Propagate the NuBuild annotations. foreach (InOutMapping mapping in this.getInOutMappings()) { if (mapping.dfysource != null && this.transformFilterAccepts(mapping.dfysource)) { AnnotationScanner.transferAnnotations( workingDirectory, mapping.dfysource, mapping.basmIfc, BoogieAsmDepBase.CommentSymbol); AnnotationScanner.transferAnnotations( workingDirectory, mapping.dfysource, mapping.basmImp, BoogieAsmDepBase.CommentSymbol); } } return(new Fresh()); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { if (this.verificationRequest.isComplete()) { VerificationResult vr = VerificationResult.fromXmlFile(this.batchSummaryVerb.getOutputFile()); if (!vr.pass) { Util.Assert(false); // Should never get here, since Ironclad app should fail before producing a verified exe. return(new VerbSyncWorker(workingDirectory, new Failed())); } } // Copy the AppLoader binary and the bootloader into the same directory as the app's binary, so the pxe-loader can find them. // REVIEW: Not clear this is doing the right thing with shift to WorkingDirectory. File.Copy(workingDirectory.PathTo(this.loaderVerb.getExe()), workingDirectory.PathTo(this.loaderCopy), true); File.Copy(workingDirectory.PathTo(this.appVerb.getExe()), workingDirectory.PathTo(this.appExecutableCopy), true); File.Copy(workingDirectory.PathTo(this.bootloader), workingDirectory.PathTo(this.bootloaderCopy), true); this.writeBootFile(workingDirectory); return(new VerbSyncWorker(workingDirectory, new Fresh())); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { // Mimic this line from src\Checked\Nucleus\Main\build.ps1: // _cat -out $OBJ\EntryCP_i.bpl -in $OBJ\MainCP_i.bpl,$SPEC_OBJ\EntryCP_i.bpl // TODO: eliminate this special-case workaround. try { // This is the trusted bit, creating the stitched ifc file. ////IEnumerable<string> ifcImports = extractImportStatements(dafnyMainIfcInput); string[] mainLines = File.ReadAllLines(workingDirectory.PathTo(this.mainBeatVerb.getOutputs().First())); string[] entryLines = File.ReadAllLines(workingDirectory.PathTo(this.genericStitch)); int sentinel_index = Array.IndexOf(entryLines, SENTINEL_APP_SPECIFIC_GOES_HERE); if (sentinel_index < 0) { throw new UserError("Sentinel string missing in " + this.genericStitch); } IEnumerable <string> entryPrologue = entryLines.Take(sentinel_index + 1); IEnumerable <string> entryEpilogue = entryLines.Skip(sentinel_index + 1); string[] appSpecificLines = File.ReadAllLines(workingDirectory.PathTo(this.appSpecificStitch)); ////File.WriteAllLines(getIfcOutput().getFilesystemPath(), ifcImports.Concat(mainLines.Concat(entryLines))); File.WriteAllLines( workingDirectory.PathTo(this.getIfcOutput()), mainLines.Concat(entryPrologue).Concat(appSpecificLines).Concat(entryEpilogue)); // Here's some (at least untrusted) workaround, snarfing and repurposing the // import list from dafny_Main_i up to Entry.imp. IEnumerable <string> impImports = extractImportStatements(workingDirectory, this.dafnyMainImpInput); string[] intext = File.ReadAllLines(workingDirectory.PathTo(this.entryImpInput)); File.WriteAllLines(workingDirectory.PathTo(this.getEntryImpOutput()), impImports.Concat(intext)); return(new VerbSyncWorker(workingDirectory, new Fresh())); } catch (IOException ex) { return(new VerbSyncWorker(workingDirectory, new Failed(ex.ToString()))); } }
public override void preprocess(WorkingDirectory workingDirectory) { base.preprocess(workingDirectory); File.Copy(workingDirectory.PathTo(basmIn), workingDirectory.PathTo(getTmpInputFile()), true); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { Disposition disposition = new Fresh(); if (this.verifyVerb != null) { VerificationResult verificationResult = VerificationResult.fromXmlFile(this.verifyVerb.getOutputs().Single()); if (!verificationResult.pass) { disposition = new Failed(); } } if (!(disposition is Failed)) { foreach (var o in this.buildVerb.getOutputs()) { if (o.getExtension() == ".exe") { File.Copy(workingDirectory.PathTo(o), workingDirectory.PathTo(this.exeOutput), overwrite: true); } else { var dest = this.RelocateBuildObjectToExeDirectory(o); File.Copy( workingDirectory.PathTo(o), workingDirectory.PathTo(dest), overwrite: true); } } } return new VerbSyncWorker(workingDirectory, disposition); }
public Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { VerificationResult vr = new VerificationResult( this.dfysource.getRelativePath(), cpuTimeSeconds, stdout, stderr, new VerificationResultDafnyParser()); vr.addBasicPresentation(); vr.toXmlFile(workingDirectory.PathTo(this.getOutputFile())); this.setWasRejectableFailure(!vr.pass); return disposition; }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { // Read and aggregate all the input results. int parseFailures = 0; int verificationFailures = 0; int timeouts = 0; int filesWithParseFailures = 0; int filesWithVerificationFailures = 0; int filesWithTimeouts = 0; int passCount = 0; int failCount = 0; double cpuTime = 0; string failedFiles = ""; string timeoutFiles = ""; string parseFiles = ""; List <VerificationMessage> failMessages = new List <VerificationMessage>(); List <VerificationResult> results = new List <VerificationResult>(); // REVIEW: Why pull out the enumerator this way? IEnumerable <BuildObject> verificationResultsEnumerator = this.verificationResults; foreach (BuildObject verificationResult in verificationResultsEnumerator) { VerificationResult vr = VerificationResult.fromXmlFile(verificationResult); results.Add(vr); if (vr == null) { return(new VerbSyncWorker( workingDirectory, new Failed("Build system broke: missing VerificationResult for " + verificationResult))); } if (vr.pass) { passCount += 1; } else { failCount += 1; failMessages.AddRange(vr.getMessages()); } parseFailures += vr.parseFailures; verificationFailures += vr.verificationFailures; timeouts += vr.timeouts; filesWithParseFailures += vr.parseFailures > 0 ? 1 : 0; filesWithVerificationFailures += vr.verificationFailures > 0 ? 1 : 0; filesWithTimeouts += vr.timeouts > 0 ? 1 : 0; if (vr.verificationFailures > 0) { failedFiles = failedFiles + vr.sourceLabel + "\n"; } if (vr.timeouts > 0) { timeoutFiles = timeoutFiles + vr.sourceLabel + "\n"; } if (vr.parseFailures > 0) { parseFiles = parseFiles + vr.sourceLabel + "\n"; } ////Logger.WriteLine("Synthesizing cpuTime from " + verificationResult); cpuTime += vr.cpuTime; } bool allPass = failCount == 0; PresentationBuilder pr = new PresentationBuilder(); int any_failures = parseFailures + verificationFailures + timeouts; string overall_status = any_failures > 0 ? "Fail" : "Success"; pr.pre(VerificationResult._VERIFICATION_RESULT_PLACEHOLDER + "\n"); pr.spacer(); pr.startHeader(); pr.color(this.colorByFailureCount(any_failures), "Overall status: " + overall_status); pr.endHeader(); pr.line("Count of files with failures: " + failCount); pr.startBullet(); pr.color(this.colorByFailureCount(filesWithParseFailures), "Files with parse failures: " + filesWithParseFailures.ToString()); pr.endBullet(); pr.startBullet(); pr.color(this.colorByFailureCount(filesWithVerificationFailures), "Files with verification failures: " + filesWithVerificationFailures.ToString()); pr.endBullet(); pr.startBullet(); pr.color(this.colorByFailureCount(filesWithVerificationFailures), "File Names with verification failures: " + failedFiles); pr.endBullet(); pr.startBullet(); pr.color(this.colorByFailureCount(filesWithTimeouts), "Files with timeouts: " + filesWithTimeouts.ToString()); pr.endBullet(); pr.startBullet(); pr.color(this.colorByFailureCount(filesWithVerificationFailures), "File Names with timeout failures: " + timeoutFiles); pr.endBullet(); pr.startBullet(); pr.color(this.colorByFailureCount(filesWithParseFailures), "File Names with parse failures: " + parseFiles); pr.endBullet(); pr.spacer(); pr.header(string.Format("Total processing time: {0:0.0}s {1}", cpuTime, new TimeSpan((long)(cpuTime * 10000000L)))); int top_n = 10; pr.header(string.Format("Slowest {0} verifications:", top_n)); results.Sort(this.ByCpuTimeDecreasing); foreach (VerificationResult slowResult in results.Take(top_n)) { pr.startBullet(); pr.color( this.colorByFailureCount(slowResult.pass ? 0 : 1), string.Format("{0,6:##0.0}s {1}", slowResult.cpuTime, slowResult.sourceLabel)); pr.endBullet(); } foreach (VerificationMessage message in failMessages) { pr.spacer(); pr.header("Failure with " + message.SourceLabel); pr.pre(message.Message); } Presentation pres = pr.fix(); VerificationResult outvr = new VerificationResult("summary", allPass, cpuTime, parseFailures, verificationFailures, timeouts, failMessages); outvr.addXmlFiller(pres); outvr.toXmlFile(workingDirectory.PathTo(this.outputObject)); this.setWasRejectableFailure(!outvr.pass); return(new VerbSyncWorker(workingDirectory, new Fresh())); }
private void writeBootFile(WorkingDirectory workingDirectory) { List<string> lines = new List<string>(); lines.Add(this.mkBootFileEntry(workingDirectory, this.loaderCopy)); lines.Add(this.mkBootFileEntry(workingDirectory, this.appExecutableCopy)); File.WriteAllLines(workingDirectory.PathTo(this.bootIniFile), lines); }
public Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { if (disposition is Failed) { return disposition; } HashSet<string> createdFiles = new HashSet<string>(Directory.GetFiles(workingDirectory.PathTo(this.getAbsDestPath())).Select(path => Path.GetFileName(path))); HashSet<string> expectedFiles = new HashSet<string>(this.getOutputs().Select(obj => obj.getFileName())); // DafnyCC/DafnySpec process a big batch of files together. Did we correctly understand what it did? if (!createdFiles.SetEquals(expectedFiles)) { // REVIEW: These are never used by anything. Remove? bool dummy = createdFiles.SetEquals(expectedFiles); int missing = expectedFiles.Except(createdFiles).Count(); int extra = createdFiles.Except(expectedFiles).Count(); string msg = "Missing files: " + string.Join(",", expectedFiles.Except(createdFiles)) + "\n" + " Extra files: " + string.Join(",", createdFiles.Except(expectedFiles)); return new Failed(msg); } // Propagate the NuBuild annotations. foreach (InOutMapping mapping in this.getInOutMappings()) { if (mapping.dfysource != null && this.transformFilterAccepts(mapping.dfysource)) { AnnotationScanner.transferAnnotations( workingDirectory, mapping.dfysource, mapping.basmIfc, BoogieAsmDepBase.CommentSymbol); AnnotationScanner.transferAnnotations( workingDirectory, mapping.dfysource, mapping.basmImp, BoogieAsmDepBase.CommentSymbol); } } return new Fresh(); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { if (this.verificationRequest.isComplete()) { VerificationResult vr = VerificationResult.fromXmlFile(this.batchSummaryVerb.getOutputFile()); if (!vr.pass) { Util.Assert(false); // Should never get here, since Ironclad app should fail before producing a verified exe. return new VerbSyncWorker(workingDirectory, new Failed()); } } // Copy the AppLoader binary and the bootloader into the same directory as the app's binary, so the pxe-loader can find them. // REVIEW: Not clear this is doing the right thing with shift to WorkingDirectory. File.Copy(workingDirectory.PathTo(this.loaderVerb.getExe()), workingDirectory.PathTo(this.loaderCopy), true); File.Copy(workingDirectory.PathTo(this.appVerb.getExe()), workingDirectory.PathTo(this.appExecutableCopy), true); File.Copy(workingDirectory.PathTo(this.bootloader), workingDirectory.PathTo(this.bootloaderCopy), true); this.writeBootFile(workingDirectory); return new VerbSyncWorker(workingDirectory, new Fresh()); }
/// <summary> /// Initializes a new instance of the CloudSubmitter class. /// </summary> /// <param name="requestIdentifier"> /// Unique identifier for this request. /// </param> /// <param name="workingDirectory"> /// Working directory the process is started in. /// </param> /// <param name="inputFiles"> /// List of input files expected by the process. /// </param> /// <param name="outputFiles"> /// List of potential output files generated by the process. /// </param> /// <param name="executable">Executable to run.</param> /// <param name="args"> /// Command line arguments to provide to the executable. /// </param> /// <param name="failureBase">Not sure what this is -- some debugging/diagnostic thing.</param> /// <param name="captureStdout">Where to (optionally) store the captured standard out.</param> /// <param name="dbgText">Debugging text for something or another.</param> public CloudSubmitter( string requestIdentifier, WorkingDirectory workingDirectory, IEnumerable<BuildObject> inputFiles, IEnumerable<BuildObject> outputFiles, string executable, string[] args, BuildObject failureBase, BuildObject captureStdout = null, string dbgText = null) { // Catch bad verb authors before they hurt themselves. Util.Assert(!executable.Contains(":")); // Hey, this looks like an absolute path! Use .getRelativePath(); it makes your output more stable. foreach (string arg in args) { // Pardon my distasteful heuristic to avoid flagging /flag:value args. Util.Assert(arg.Length < 2 || arg[1] != ':'); // Hey, this looks like an absolute path! Use .getRelativePath() to tolerate crossing machine boundaries. } // Stash away things we'll want to remember later. this.workingDirectory = workingDirectory; // Create list of input file mappings. // REVIEW: We're not running on the main thread at this point. Are below calls all thread-safe? List<BuildObjectValuePointer> inputFileMappings = new List<BuildObjectValuePointer>(); foreach (BuildObject file in inputFiles) { string fileHash = BuildEngine.theEngine.Repository.GetHash(file); Util.Assert(!string.IsNullOrEmpty(fileHash)); inputFileMappings.Add(new BuildObjectValuePointer(fileHash, file.getRelativePath())); // Ensure that the input files are in the cloud cache. // REVIEW: best way to determine this is a source file? ItemCacheContainer container; if (file is SourcePath) { container = ItemCacheContainer.Sources; } else { container = ItemCacheContainer.Objects; } ItemCacheMultiplexer multiplexedCache = BuildEngine.theEngine.ItemCache as ItemCacheMultiplexer; Util.Assert(multiplexedCache != null); multiplexedCache.SyncItemToCloud(container, fileHash); } // Prepare cloud execution request for submission. string arguments = string.Join(" ", args); CloudExecutionRequest request = new CloudExecutionRequest( BuildEngine.theEngine.CloudReportQueueName, requestIdentifier, CloudExecutionRequest.Operation.RunExecutable, executable, arguments, inputFileMappings, outputFiles); BuildEngine.theEngine.CloudExecutionQueue.SubmitRequest(request); // Wait for remote execution to finish. int requestsOutstanding; Console.WriteLine("Waiting on remote execution report for request '{0}'.", requestIdentifier); CloudExecutionReport executionReport = BuildEngine.theEngine.CloudExecutionQueue.GetReport(requestIdentifier, out requestsOutstanding); Console.WriteLine("Received remote execution report for request '{0}'. {1} others still outstanding.", requestIdentifier, requestsOutstanding); // Record what we got back. this.exitCode = executionReport.ExitCode; this.stdout = executionReport.StandardOutput; this.stderr = executionReport.StandardError; this.cpuTime = executionReport.CpuTime; // Copy output files from cloud cache back to local working dir. // REVIEW: This is just to set things up as expected for the // Scheduler's recordResult routine. Could re-architect this to // be more efficient for the remote execution case. foreach (BuildObjectValuePointer outputFileMapping in executionReport.OutputFileMappings) { BuildEngine.theEngine.CloudCache.FetchItemToFile( ItemCacheContainer.Objects, outputFileMapping.ObjectHash, workingDirectory.PathTo(outputFileMapping.RelativePath)); } }
public virtual Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { if (!(disposition is Failed)) { // Check that the executable isn't too large. long exeSize = new FileInfo(workingDirectory.PathTo(this.outputObject)).Length; if (exeSize > this.maxExeSize) { return new Failed("Executable too big"); } } return disposition; }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { // Mimic this line from src\Checked\Nucleus\Main\build.ps1: // _cat -out $OBJ\EntryCP_i.bpl -in $OBJ\MainCP_i.bpl,$SPEC_OBJ\EntryCP_i.bpl // TODO: eliminate this special-case workaround. try { // This is the trusted bit, creating the stitched ifc file. ////IEnumerable<string> ifcImports = extractImportStatements(dafnyMainIfcInput); string[] mainLines = File.ReadAllLines(workingDirectory.PathTo(this.mainBeatVerb.getOutputs().First())); string[] entryLines = File.ReadAllLines(workingDirectory.PathTo(this.genericStitch)); int sentinel_index = Array.IndexOf(entryLines, SENTINEL_APP_SPECIFIC_GOES_HERE); if (sentinel_index < 0) { throw new UserError("Sentinel string missing in " + this.genericStitch); } IEnumerable<string> entryPrologue = entryLines.Take(sentinel_index + 1); IEnumerable<string> entryEpilogue = entryLines.Skip(sentinel_index + 1); string[] appSpecificLines = File.ReadAllLines(workingDirectory.PathTo(this.appSpecificStitch)); ////File.WriteAllLines(getIfcOutput().getFilesystemPath(), ifcImports.Concat(mainLines.Concat(entryLines))); File.WriteAllLines( workingDirectory.PathTo(this.getIfcOutput()), mainLines.Concat(entryPrologue).Concat(appSpecificLines).Concat(entryEpilogue)); // Here's some (at least untrusted) workaround, snarfing and repurposing the // import list from dafny_Main_i up to Entry.imp. IEnumerable<string> impImports = extractImportStatements(workingDirectory, this.dafnyMainImpInput); string[] intext = File.ReadAllLines(workingDirectory.PathTo(this.entryImpInput)); File.WriteAllLines(workingDirectory.PathTo(this.getEntryImpOutput()), impImports.Concat(intext)); return new VerbSyncWorker(workingDirectory, new Fresh()); } catch (IOException ex) { return new VerbSyncWorker(workingDirectory, new Failed(ex.ToString())); } }
public Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition) { var dis = disposition; string cSharpPath = Path.ChangeExtension(workingDirectory.PathTo(this.expandedSource), CSharpExt); if (!File.Exists(cSharpPath)) { // Dafny has a bug where compilation fails but result code is still 0. dis = new Failed(); } if (dis is Fresh) { this.RewriteCSharpFile(cSharpPath, workingDirectory.PathTo(this.output)); } return dis; }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { // If we were given an optional dafny input, let this build know about the resulting verb's output. // Note: this is deliberately written to break if someone changes DafnyCompileOneVerb to have multiple outputs. if (this.dafnyCompileOneVerb != null) { File.Copy( workingDirectory.PathTo(this.dafnyCompileOneVerb.getOutputs().Single()), Path.Combine(workingDirectory.PathTo(this.outputPathSuffix), "DafnyDerivedInput.cs")); } List<string> args = new List<string>(); args.Add(string.Format("/p:OutDir={0}", workingDirectory.PathTo(this.outputPathSuffix))); args.Add(string.Format("/p:Configuration={0}", this.releaseBuild ? "Release" : "Debug")); ////args.Add("/fileLogger"); // Uncomment to log MSBuild execution. args.Add(workingDirectory.PathTo(this.solutionFile)); // TODO: Fix absolute path to MSBuild.exe (at least use %SystemRoot%)! return new ProcessInvokeAsyncWorker( workingDirectory, this, "c:\\Windows\\Microsoft.NET\\Framework\\v4.0.30319\\MSBuild.exe", args.ToArray(), ProcessExitCodeHandling.NonzeroIsFailure, failureBase: getDiagnosticsBase(), allowAbsoluteExe: true, allowAbsoluteArgs: true); }
public static void propagatePrivateImports( WorkingDirectory workingDirectory, IContextGeneratingVerb contextVerb, BuildObject srcobj, BuildObject dstobj) { // Rewrite basm output to propagate any import statements from the beat file. // TODO this step really should be a beat function, not part of the build system. IEnumerable<BeatIncludes.LabeledInclude> beatImports = getBeatFlavoredShallowIncludesLabeled(contextVerb, srcobj); StringBuilder sb = new StringBuilder(); foreach (BeatIncludes.LabeledInclude li in beatImports) { sb.Append("//-"); sb.Append(li.importFilter == BeatIncludes.ImportFilter.ForBasmOnly ? "private-basmonly-import" : "private-import"); sb.Append(" "); sb.Append(li.buildObject.getFileNameWithoutExtension()); sb.AppendLine(";"); } // REVIEW: Improve upon this round-about way of prepending to a file? string beatOutput = File.ReadAllText(workingDirectory.PathTo(dstobj)); File.Delete(workingDirectory.PathTo(dstobj)); File.WriteAllText(workingDirectory.PathTo(dstobj), sb.ToString() + beatOutput); }
public void store(WorkingDirectory workingDirectory, BuildObject location) { this.complete = true; string[] lines = this.verificationObligations.Select(vo => vo.getRelativePath()).ToArray(); File.WriteAllLines(workingDirectory.PathTo(location), lines); }
// TODO: Rename obj to something meaningful. Is it a boot file? private string mkBootFileEntry(WorkingDirectory workingDirectory, BuildObject obj) { return string.Format("Size={0} Path=/{1}", new FileInfo(workingDirectory.PathTo(obj)).Length, obj.getFileName()); }
// TODO: Rename obj to something meaningful. Is it a boot file? private string mkBootFileEntry(WorkingDirectory workingDirectory, BuildObject obj) { return(string.Format("Size={0} Path=/{1}", new FileInfo(workingDirectory.PathTo(obj)).Length, obj.getFileName())); }
private static IEnumerable<string> extractImportStatements(WorkingDirectory workingDirectory, BuildObject obj) { // Well, it might be nice to use BeatExtensions.propagatePrivateImports, but that requires // a context to interpret the input imports. We don't really want to cons up yet ANOTHER // intermediate context, so here's a temporary workaround. Caution; may be brittle. IEnumerable<string> imports = File.ReadAllLines(workingDirectory.PathTo(obj)) .Where(line => line.Contains("private-import")); // Note that dafny_Main_i didn't really expect us to steal its // imports, so it hasn't conditioned the Checked and Trusted imports to be beat-resistant. imports = imports.Select( line => line.Contains("Checked") || line.Contains("Trusted") ? line.Replace("private-import", "private-basmonly-import") : line); return imports; }
/// <summary> /// Run the requested executable and produce a report of the results. /// </summary> /// <param name="executionRequest">The execution request.</param> /// <returns>A report of the results.</returns> private CloudExecutionReport RunAnExecutable(CloudExecutionRequest executionRequest) { // REVIEW: How/whether to use this. BuildObject diagnosticsBase = new BuildObject(Path.Combine("nuobj", "diagnostics", "process")); // Prep working directory with input files and output dirs. // TODO: The below will throw cache exceptions if something // isn't there (they should all be though). Need to catch // these and fail the execution request when this happens. WorkingDirectory workingDirectory = new WorkingDirectory(this.virtualIronRoot); foreach (BuildObjectValuePointer inputFile in executionRequest.InputFileMappings) { // REVIEW: How to determine cache container here. ItemCacheContainer container = ItemCacheContainer.Sources; if (this.multiplexedItemCache.GetItemSize(container, inputFile.ObjectHash) == -1) { container = ItemCacheContainer.Objects; } // TODO: Move path/directory manipulation code into // WorkingDirectory and/or ItemCache. string inputFilePath = workingDirectory.PathTo(inputFile.RelativePath); Directory.CreateDirectory(Path.GetDirectoryName(inputFilePath)); // REVIEW: Still neeeded? this.multiplexedItemCache.FetchItemToFile( container, inputFile.ObjectHash, inputFilePath); } foreach (BuildObject outputFile in executionRequest.OutputFiles) { workingDirectory.CreateDirectoryFor(outputFile); } // Run executable. ProcessInvoker pinv = new ProcessInvoker( workingDirectory, executionRequest.Executable, new string[] { executionRequest.Arguments }, diagnosticsBase, null, // This is captureStdout. TODO: Should cleanup how this is used in ProcessInvoker. null); // This is dbgText. REVIEW: How/whether to use this. // When ProcessInvoker's constructor returns, the process has // finished running. Console.WriteLine("Request {0} completed in {1} seconds.", executionRequest.Identifier, pinv.CpuTime); // Store output files in the (cloud) item cache, and create a // list of the mappings. List<BuildObjectValuePointer> outputFileMappings = new List<BuildObjectValuePointer>(); foreach (BuildObject outFile in executionRequest.OutputFiles) { if (File.Exists(workingDirectory.PathTo(outFile))) { string fileHash = Util.hashFilesystemPath(workingDirectory.PathTo(outFile)); Util.Assert(!string.IsNullOrEmpty(fileHash)); // Note we explicitly write to the cloud cache here. this.cloudCache.StoreItemFromFile(ItemCacheContainer.Objects, fileHash, workingDirectory.PathTo(outFile)); outputFileMappings.Add(new BuildObjectValuePointer(fileHash, outFile.getRelativePath())); } } // Collect the results into a report. CloudExecutionReport report = new CloudExecutionReport( executionRequest.Identifier, CloudExecutionReport.StatusCode.Completed, pinv.ExitCode, pinv.GetStdout(), pinv.GetStderr(), pinv.CpuTime, outputFileMappings); return report; }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { if (false) { #pragma warning disable 162 File.WriteAllText(workingDirectory.PathTo(getOutputFile()), "Verification disabled temporarily for debugging"); return new VerbSyncWorker(workingDirectory, new Fresh()); #pragma warning restore 162 } List<string> args = new List<string>(); args.Add("/noinfer"); args.Add("/typeEncoding:m"); args.Add("/z3opt:ARITH_RANDOM_SEED=1"); args.Add("/timeLimit:" + timeLimit); args.AddRange(getFlags()); args.Add(bplInput.getRelativePath()); return new ProcessInvokeAsyncWorker( workingDirectory, this, getBoogieExecutable().getRelativePath(), args.ToArray(), ProcessExitCodeHandling.NonzeroIsOkay, getDiagnosticsBase(), allowCloudExecution: true, returnStandardOut: true, returnStandardError: true); }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { // Read and aggregate all the input results. int parseFailures = 0; int verificationFailures = 0; int timeouts = 0; int filesWithParseFailures = 0; int filesWithVerificationFailures = 0; int filesWithTimeouts = 0; int passCount = 0; int failCount = 0; double cpuTime = 0; List<VerificationMessage> failMessages = new List<VerificationMessage>(); List<VerificationResult> results = new List<VerificationResult>(); // REVIEW: Why pull out the enumerator this way? IEnumerable<BuildObject> verificationResultsEnumerator = this.verificationResults; foreach (BuildObject verificationResult in verificationResultsEnumerator) { VerificationResult vr = VerificationResult.fromXmlFile(verificationResult); results.Add(vr); if (vr == null) { return new VerbSyncWorker( workingDirectory, new Failed("Build system broke: missing VerificationResult for " + verificationResult)); } if (vr.pass) { passCount += 1; } else { failCount += 1; failMessages.AddRange(vr.getMessages()); } parseFailures += vr.parseFailures; verificationFailures += vr.verificationFailures; timeouts += vr.timeouts; filesWithParseFailures += vr.parseFailures > 0 ? 1 : 0; filesWithVerificationFailures += vr.verificationFailures > 0 ? 1 : 0; filesWithTimeouts += vr.timeouts > 0 ? 1 : 0; ////Logger.WriteLine("Synthesizing cpuTime from " + verificationResult); cpuTime += vr.cpuTime; } bool allPass = failCount == 0; PresentationBuilder pr = new PresentationBuilder(); int any_failures = parseFailures + verificationFailures + timeouts; string overall_status = any_failures > 0 ? "Fail" : "Success"; pr.pre(VerificationResult._VERIFICATION_RESULT_PLACEHOLDER+"\n"); pr.spacer(); pr.startHeader(); pr.color(this.colorByFailureCount(any_failures), "Overall status: " + overall_status); pr.endHeader(); pr.line("Count of files with failures: " + failCount); pr.startBullet(); pr.color(this.colorByFailureCount(filesWithParseFailures), "Files with parse failures: " + filesWithParseFailures.ToString()); pr.endBullet(); pr.startBullet(); pr.color(this.colorByFailureCount(filesWithVerificationFailures), "Files with verification failures: " + filesWithVerificationFailures.ToString()); pr.endBullet(); pr.startBullet(); pr.color(this.colorByFailureCount(filesWithTimeouts), "Files with timeouts: " + filesWithTimeouts.ToString()); pr.endBullet(); pr.spacer(); pr.header(string.Format("Total processing time: {0:0.0}s {1}", cpuTime, new TimeSpan((long)(cpuTime * 10000000L)))); int top_n = 10; pr.header(string.Format("Slowest {0} verifications:", top_n)); results.Sort(this.ByCpuTimeDecreasing); foreach (VerificationResult slowResult in results.Take(top_n)) { pr.startBullet(); pr.color( this.colorByFailureCount(slowResult.pass ? 0 : 1), string.Format("{0,6:##0.0}s {1}", slowResult.cpuTime, slowResult.sourceLabel)); pr.endBullet(); } foreach (VerificationMessage message in failMessages) { pr.spacer(); pr.header("Failure with " + message.SourceLabel); pr.pre(message.Message); } Presentation pres = pr.fix(); VerificationResult outvr = new VerificationResult("summary", allPass, cpuTime, parseFailures, verificationFailures, timeouts, failMessages); outvr.addXmlFiller(pres); outvr.toXmlFile(workingDirectory.PathTo(this.outputObject)); this.setWasRejectableFailure(!outvr.pass); return new VerbSyncWorker(workingDirectory, new Fresh()); }
/// <summary> /// Initializes a new instance of the ProcessInvoker class. /// </summary> /// <param name="workingDirectory">The working directory the process is started in.</param> /// <param name="executable">The executable to run.</param> /// <param name="args">The command line arguments to provide to the executable.</param> /// <param name="failureBase">Not sure what this is -- some debugging/diagnostic thing.</param> /// <param name="captureStdout">Where to (optionally) capture standard out.</param> /// <param name="dbgText">Debugging text for something or another.</param> /// <param name="allowAbsoluteExe">Whether to allow an absolute (rather than relative) file path to the executable.</param> /// <param name="allowAbsoluteArgs">Whether to allow absolute (rather than relative) file paths as arguments.</param> /// <param name="workingDirOverride">The working directory to use.</param> public ProcessInvoker( WorkingDirectory workingDirectory, string executable, string[] args, BuildObject failureBase, BuildObject captureStdout = null, string dbgText = null, bool allowAbsoluteExe = false, bool allowAbsoluteArgs = false, string workingDirOverride = null) { // Catch bad verb authors before they hurt themselves. Util.Assert(allowAbsoluteExe || !executable.Contains(":")); // Hey, this looks like an absolute path! Use .getRelativePath(); it makes your output more stable. foreach (string arg in args) { // Pardon my distasteful heuristic to avoid flagging /flag:value args. Util.Assert(allowAbsoluteArgs || arg.Length < 2 || arg[1] != ':'); // Hey, this looks like an absolute path! Use .getRelativePath() to tolerate crossing machine boundaries. } this.workingDirectory = workingDirectory; this.stdout = new StringBuilder(); this.stderr = new StringBuilder(); using (Job job = new Job()) { using (Process proc = new Process()) { if (allowAbsoluteExe) { proc.StartInfo.FileName = executable; } else { // TODO: *All* async verbs need to list their executable (and all the libs it depends upon) as dependencies. proc.StartInfo.FileName = workingDirectory.PathTo(executable); } // TODO Is there a better way to escape the args to avoid problems with spaces? proc.StartInfo.Arguments = string.Join(" ", args); proc.StartInfo.WorkingDirectory = workingDirOverride == null ? workingDirectory.Root : workingDirOverride; proc.StartInfo.RedirectStandardOutput = true; // REVIEW: Maybe we should always capture stdout in a StringBuilder and just write it out to a file afterwards if requested? if (captureStdout != null) { this.tmpStdout = new BuildObject(captureStdout.getRelativePath() + ".tmp"); this.stdoutFile = new StreamWriter(workingDirectory.PathTo(this.tmpStdout)); proc.OutputDataReceived += new DataReceivedEventHandler(this.StdoutRedirectHandler); } else { // Collect stdout here for diagnostics. proc.OutputDataReceived += new DataReceivedEventHandler(this.StdoutHandler); } proc.StartInfo.RedirectStandardError = true; proc.ErrorDataReceived += new DataReceivedEventHandler(this.StderrHandler); proc.StartInfo.UseShellExecute = false; string commandLine = proc.StartInfo.FileName + " " + proc.StartInfo.Arguments; if (failureBase != null && AlwaysEmitDiagnostics) { // In diagnostic mode, we emit the command line twice, once ahead in case Boogie decides // to run away and never come back. BuildObject failureBatObj = failureBase.makeOutputObject(".bat"); workingDirectory.CreateDirectoryFor(failureBatObj); File.WriteAllText(workingDirectory.PathTo(failureBatObj), commandLine); } proc.Start(); job.AddProcess(proc); proc.BeginOutputReadLine(); proc.BeginErrorReadLine(); proc.WaitForExit(); this.cpuTime = job.GetCpuTime().TotalSeconds; this.exitCode = proc.ExitCode; if (this.stdoutFile != null) { this.stdoutFile.Close(); } if (failureBase != null && AlwaysEmitDiagnostics) { workingDirectory.CreateDirectoryFor(failureBase); File.WriteAllText(workingDirectory.PathTo(failureBase.makeOutputObject(".bat")), commandLine); File.WriteAllText(workingDirectory.PathTo(failureBase.makeOutputObject(".txt")), dbgText); File.WriteAllText(workingDirectory.PathTo(failureBase.makeOutputObject(".stdout")), this.GetStdoutString()); File.WriteAllText(workingDirectory.PathTo(failureBase.makeOutputObject(".stderr")), this.GetStderr()); } } } // REVIEW: Add Delete, Exists and Move methods to WorkingDirectory class? if (this.tmpStdout != null && File.Exists(workingDirectory.PathTo(this.tmpStdout))) { // REVIEW: Nothing should be here. Bother with the delete? File.Delete(workingDirectory.PathTo(captureStdout)); File.Move(workingDirectory.PathTo(this.tmpStdout), workingDirectory.PathTo(captureStdout)); this.tmpStdout = null; } }
/// <summary> /// Record how each output of a task appeared. /// </summary> /// <param name="completion">The task completion notification.</param> /// <returns>Overall result of the verb execution.</returns> private Disposition recordResult(VerbRunner.TaskCompletion completion) { WorkingDirectory workingDirectory = completion.workingDirectory; IVerb verb = completion.verb; Disposition executionDisposition = completion.disposition; List <BuildObjectValuePointer> outputs = new List <BuildObjectValuePointer>(); List <string> missingOutputs = new List <string>(); IEnumerable <BuildObject> expectedOutputs = verb.getOutputs(); if (executionDisposition is Failed) { expectedOutputs = expectedOutputs.Concat(verb.getFailureOutputs()); } bool hasVirtualOutputs = false; foreach (BuildObject outobj in expectedOutputs) { if (!(outobj is VirtualBuildObject)) { // For expected file outputs, check for existence in working directory. // REVIEW: Add method to WorkingDirectory which does this? if (File.Exists(workingDirectory.PathTo(outobj))) { // Try to catch accidental case mismatches that would burn us when we // try to fetch the file back in. ////string fsname = PathNormalizer.dbg_normalizePath_nocache(outobj.deprecatedGetFilesystemPath(), false); ////Util.Assert(Path.GetFileName(fsname).Equals(outobj.getFileName())); // REVIEW: Do we need to worry about case mismatches anymore? See comments above. outputs.Add(this.repository.Store(workingDirectory, outobj, executionDisposition)); // Store a copy of this verb output as a file in the real nuobj directory. Util.Assert(outobj.getRelativePath().StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal)); string nuobjPath = IronRootDirectory.PathTo(outobj); Directory.CreateDirectory(Path.GetDirectoryName(nuobjPath)); File.Copy(workingDirectory.PathTo(outobj), nuobjPath, true); } else { missingOutputs.Add(string.Format("Missing expected output {0}", outobj.getRelativePath())); } } else { hasVirtualOutputs = true; if (this.repository.GetDisposition(outobj) is Fresh) { // Nothing to cache; virtual objects only survive in the Repository, the in-process store. } else { missingOutputs.Add(string.Format("Missing expected virtual {0}", outobj.getRelativePath())); } } } if (!(executionDisposition is Failed) && missingOutputs.Count() > 0) { executionDisposition = new Failed(missingOutputs); } ResultSummaryRecord summary = new ResultSummaryRecord(verb, executionDisposition, outputs); string inputHash = this.computeInputHash(verb, true); Util.Assert(inputHash != null); if (!hasVirtualOutputs) { this.repository.StoreResult(inputHash, summary); } else { this.Say("Not caching verb persistently: " + verb); } this.verbIsComplete(verb, executionDisposition); return(executionDisposition); }
/// <summary> /// Expand a dafny source file to include all of its includes inline. /// </summary> /// <param name="workingDirectory">Temporary working directory to use.</param> /// <param name="input">Source build object.</param> /// <param name="output">Where to create build object for expanded source file.</param> public void ExpandDafny(WorkingDirectory workingDirectory, SourcePath input, SourcePath output) { // Prepare the output stream. using (TextWriter outWriter = new StreamWriter(workingDirectory.PathTo(output))) { // Stash away a few things for use by our recursive helper function. this.workingDirectory = workingDirectory; this.outputWriter = outWriter; this.visited = new HashSet<string>(); // Recursively expand the initial Dafny source file to inline all of its includes. this.ExpandDafnyRecurse(input); } // Cache the output file in the Repository. BuildEngine.theEngine.Repository.Store(workingDirectory, output, new Fresh()); }
/// <summary> /// Initializes a new instance of the CloudSubmitter class. /// </summary> /// <param name="requestIdentifier"> /// Unique identifier for this request. /// </param> /// <param name="workingDirectory"> /// Working directory the process is started in. /// </param> /// <param name="inputFiles"> /// List of input files expected by the process. /// </param> /// <param name="outputFiles"> /// List of potential output files generated by the process. /// </param> /// <param name="executable">Executable to run.</param> /// <param name="args"> /// Command line arguments to provide to the executable. /// </param> /// <param name="failureBase">Not sure what this is -- some debugging/diagnostic thing.</param> /// <param name="captureStdout">Where to (optionally) store the captured standard out.</param> /// <param name="dbgText">Debugging text for something or another.</param> public CloudSubmitter( string requestIdentifier, WorkingDirectory workingDirectory, IEnumerable <BuildObject> inputFiles, IEnumerable <BuildObject> outputFiles, string executable, string[] args, BuildObject failureBase, BuildObject captureStdout = null, string dbgText = null) { // Catch bad verb authors before they hurt themselves. Util.Assert(!executable.Contains(":")); // Hey, this looks like an absolute path! Use .getRelativePath(); it makes your output more stable. foreach (string arg in args) { // Pardon my distasteful heuristic to avoid flagging /flag:value args. Util.Assert(arg.Length < 2 || arg[1] != ':'); // Hey, this looks like an absolute path! Use .getRelativePath() to tolerate crossing machine boundaries. } // Stash away things we'll want to remember later. this.workingDirectory = workingDirectory; // Create list of input file mappings. // REVIEW: We're not running on the main thread at this point. Are below calls all thread-safe? List <BuildObjectValuePointer> inputFileMappings = new List <BuildObjectValuePointer>(); foreach (BuildObject file in inputFiles) { string fileHash = BuildEngine.theEngine.Repository.GetHash(file); Util.Assert(!string.IsNullOrEmpty(fileHash)); inputFileMappings.Add(new BuildObjectValuePointer(fileHash, file.getRelativePath())); // Ensure that the input files are in the cloud cache. // REVIEW: best way to determine this is a source file? ItemCacheContainer container; if (file is SourcePath) { container = ItemCacheContainer.Sources; } else { container = ItemCacheContainer.Objects; } ItemCacheMultiplexer multiplexedCache = BuildEngine.theEngine.ItemCache as ItemCacheMultiplexer; Util.Assert(multiplexedCache != null); multiplexedCache.SyncItemToCloud(container, fileHash); } // Prepare cloud execution request for submission. string arguments = string.Join(" ", args); CloudExecutionRequest request = new CloudExecutionRequest( BuildEngine.theEngine.CloudReportQueueName, requestIdentifier, CloudExecutionRequest.Operation.RunExecutable, executable, arguments, inputFileMappings, outputFiles); BuildEngine.theEngine.CloudExecutionQueue.SubmitRequest(request); // Wait for remote execution to finish. int requestsOutstanding; Console.WriteLine("Waiting on remote execution report for request '{0}'.", requestIdentifier); CloudExecutionReport executionReport = BuildEngine.theEngine.CloudExecutionQueue.GetReport(requestIdentifier, out requestsOutstanding); Console.WriteLine("Received remote execution report for request '{0}'. {1} others still outstanding.", requestIdentifier, requestsOutstanding); // Record what we got back. this.exitCode = executionReport.ExitCode; this.stdout = executionReport.StandardOutput; this.stderr = executionReport.StandardError; this.cpuTime = executionReport.CpuTime; // Copy output files from cloud cache back to local working dir. // REVIEW: This is just to set things up as expected for the // Scheduler's recordResult routine. Could re-architect this to // be more efficient for the remote execution case. foreach (BuildObjectValuePointer outputFileMapping in executionReport.OutputFileMappings) { BuildEngine.theEngine.CloudCache.FetchItemToFile( ItemCacheContainer.Objects, outputFileMapping.ObjectHash, workingDirectory.PathTo(outputFileMapping.RelativePath)); } }
public override IVerbWorker getWorker(WorkingDirectory workingDirectory) { if (this.verificationRequest.isComplete()) { // If the verification succeeded, then we convert the untrusted exe into a trusted exe (via a copy). VerificationResult vr = VerificationResult.fromXmlFile(this.verifyResultsVerb.getOutputFile()); if (!vr.pass) { return new VerbSyncWorker(workingDirectory, new Failed()); } File.Copy(workingDirectory.PathTo(this.srcObject), workingDirectory.PathTo(this.outputObject), true); // True => Overwrite } else { UnverifiedSentinelVirtualContents contents = new UnverifiedSentinelVirtualContents(); BuildEngine.theEngine.Repository.StoreVirtual(this.outputObject, new Fresh(), contents); } return new VerbSyncWorker(workingDirectory, new Fresh()); }