public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
// "beat $BUILD_DEFS -out $out.tmp -in $in $incls"
List<string> args = new List<string>();
args.Add("-in");
args.Add(this.beatobj.getRelativePath());
IEnumerable<BuildObject> beatImports =
BeatExtensions.getBeatFlavoredShallowIncludes(this.contextVerb, this.beatobj, BeatIncludes.ImportFilter.ForBeatOrBasm);
foreach (BuildObject ifcObj in beatImports.Where(obj => !obj.Equals(this.beatobj)))
{
Util.Assert(!ifcObj.getRelativePath().Contains(".imp")); // Erk, don't feed imp files as includes!
args.Add("-i");
args.Add(ifcObj.getRelativePath());
}
args.AddRange(this.contextVerb.getPoundDefines().ToDefArgs());
string dbgText = string.Format(
"rem verb {0}{1}",
this,
System.Environment.NewLine);
return new ProcessInvokeAsyncWorker(
workingDirectory,
this,
this.getBeatExecutable().getRelativePath(),
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
captureStdout: this.outputFile(),
failureBase: getDiagnosticsBase(),
dbgText: dbgText);
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
List<string> args = this.getArgs();
preprocess(workingDirectory);
// We use workingDirOverride flag here to change the path that the
// process starts in to one that is still in our workingDirectory.
// So this isn't so bad. It would be better if NuBuild would only
// let us supply a relative path here, however.
string overrideDir = null;
if (this.getWorkingDir() != null)
{
overrideDir = workingDirectory.PathTo(this.getWorkingDir());
}
return new ProcessInvokeAsyncWorker(
workingDirectory,
this,
getSymDiffExecutable().getRelativePath(),
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
getDiagnosticsBase(),
workingDirOverride: overrideDir,
returnStandardOut: true);
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
VerbOutputsContext context = new VerbOutputsContext(this.parent, this.assertSuspiciousDafnyImpls);
ContextContents contents = new ContextContents(context);
BuildEngine.theEngine.Repository.StoreVirtual(this.getContextOutput(), new Fresh(), contents);
return new VerbSyncWorker(workingDirectory, new Fresh());
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
List<string> args = new List<string>();
//// args.add(BUILD_DEFS
//// args.add(boogieasm_flags)
args.Add(getAction());
BuildObject captureStdout = null;
if (outFlagWorks())
{
args.Add("-out");
args.Add(outputFile().getRelativePath());
}
else
{
captureStdout = outputFile();
}
BasmModuleAccumulator acc = new BasmModuleAccumulator(context, upstreamObj, includeAllImps());
Util.Assert(acc.ddisp == DependencyDisposition.Complete);
args.AddRange(acc.basmModules.Select(module => module.getRelativePath()));
args.AddRange(context.getPoundDefines().ToDefArgs());
extendArgs(args);
return new ProcessInvokeAsyncWorker(
workingDirectory,
this,
getBoogieasmExecutable().getRelativePath(),
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
getDiagnosticsBase(),
captureStdout: captureStdout);
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
// Scrub the "makeflags" environment variable from our environment
// so we don't pass it down to our worker process. Some users may
// have things there that conflict with our usage.
Process myProcess = System.Diagnostics.Process.GetCurrentProcess();
StringDictionary environmentVariables = myProcess.StartInfo.EnvironmentVariables;
environmentVariables.Remove("MAKEFLAGS");
List<string> args = new List<string>();
args.Add(string.Format("OBJ={0}\\obj", workingDirectory.PathTo(this.outputPathSuffix)));
args.Add(string.Format("BIN={0}", workingDirectory.PathTo(this.outputPathSuffix)));
args.Add("-f");
args.Add(workingDirectory.PathTo(this.makefile));
// TODO: Remove reliance on workingDirOverride, which currently hides dependency issues and other problems.
return new ProcessInvokeAsyncWorker(
workingDirectory,
this,
getNmakeExecutable().getRelativePath(), ////"c:/Program Files (x86)/Microsoft Visual Studio 12.0/VC/bin/nmake.exe",
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
workingDirOverride: IronRootDirectory.PathTo(this.makefile.getDirPath()),
failureBase: getDiagnosticsBase(),
//allowAbsoluteExe: true,
allowAbsoluteArgs: true);
}
/// <summary>
/// Reads a build object from the local filesystem and stores it in the
/// cache.
/// </summary>
/// <param name="workingDirectory">
/// Private directory for verb execution.
/// </param>
/// <param name="obj">The build object to store in the cache.</param>
/// <param name="disposition">
/// Disposition of verb which created this object (if known).
/// </param>
/// <returns>A BuildObjectValuePointer describing the object.</returns>
public BuildObjectValuePointer Store(WorkingDirectory workingDirectory, BuildObject obj, Disposition disposition)
{
string contentHash = Util.hashFilesystemPath(workingDirectory.PathTo(obj));
this.itemCache.StoreItemFromFile(ItemCacheContainer.Objects, contentHash, workingDirectory.PathTo(obj));
this.Add(obj, disposition, contentHash, null);
return new BuildObjectValuePointer(contentHash, obj.getRelativePath());
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
IEnumerable<BuildObject> verificationResults = getBoogieVerbs(verificationRequest)
.Select(boogie_verb => boogie_verb.getOutputFile());
VerificationObligationList vol = new VerificationObligationList(verificationResults);
vol.store(workingDirectory, this.obligations);
return new VerbSyncWorker(workingDirectory, new Fresh());
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
IEnumerable<BuildObject> verificationResults = this.getVerbs()
.Where(verb => verb is VerificationResultVerb)
.Select(dfy_one => ((VerificationResultVerb)dfy_one).getOutputFile());
VerificationObligationList vol = new VerificationObligationList(verificationResults);
vol.store(workingDirectory, this.obligations);
return new VerbSyncWorker(workingDirectory, new Fresh());
}
public override Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition)
{
if (!(disposition is Failed))
{
File.WriteAllText(workingDirectory.PathTo(this.getOutputFile()), stdout);
}
return disposition;
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
IEnumerable<IIncludePathContext> contexts = this.parents.Select(parent =>
((ContextContents)BuildEngine.theEngine.Repository.FetchVirtual(parent.getContextOutput())).Context);
ConcatContext context = new ConcatContext(contexts);
ContextContents contents = new ContextContents(context);
BuildEngine.theEngine.Repository.StoreVirtual(this.getContextOutput(), new Fresh(), contents);
return new VerbSyncWorker(workingDirectory, new Fresh());
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
// TODO: We shouldn't be using absolute paths to any of these things.
// Change this to allow VS and SDKs to be installed anywhere.
string linker = @"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\bin\link.exe";
string vc_lib_dir = @"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\lib";
string sdk_dir = @"C:\Program Files (x86)\Windows Kits\8.1\Lib\winv6.3\um\x86";
string kernel_lib = @"C:\Program Files (x86)\Windows Kits\8.1\Lib\winv6.3\um\x86\kernel32.Lib";
string standalone_support_lib = getStandaloneLib().getRelativePath();
SourcePath zero1 = new SourcePath("tools\\scripts\\zero.obj", SourcePath.SourceType.Tools);
SourcePath zero2 = new SourcePath("tools\\scripts\\zero2.obj", SourcePath.SourceType.Tools);
// TODO: Fail more gracefully? Or better yet, move these into iron/tools.
if (!Directory.Exists(vc_lib_dir))
{
throw new FileNotFoundException("Missing Visual C++ library directory: " + vc_lib_dir);
}
if (!Directory.Exists(sdk_dir) || !File.Exists(kernel_lib))
{
throw new FileNotFoundException("Missing Windows SDK libraries: " + sdk_dir + ", " + kernel_lib + @". Try installing the Windows SDK from: \\research\Root\Products\Developers\Windows Driver Kit 8.1");
}
// TODO: Unpack/generate these automatically.
// TODO: Brian, we're really not going to want to cache these big, empty sources. Or compress? All big (>10MB) files.
// are mostly zeros.
if (!File.Exists(IronRootDirectory.PathTo(zero1)) || !File.Exists(IronRootDirectory.PathTo(zero2)))
{
throw new FileNotFoundException("Missing object files of zeroes: " + zero1 + ", " + zero2 + ". Try running: tools\\scripts\\build-standalone-init.sh");
}
List<string> args = new List<string>() { "/DEBUG", "/subsystem:console", "/LARGEADDRESSAWARE", "/fixed" };
args.Add(objFile.getRelativePath());
args.Add(zero1.getRelativePath());
args.Add(zero2.getRelativePath());
args.Add(standalone_support_lib);
args.Add(@"""" + kernel_lib + @"""");
args.Add("\"/libpath:" + vc_lib_dir + '"');
args.Add("\"/libpath:" + sdk_dir + '"');
args.Add("/out:" + outputObject.getRelativePath());
args.Add("/entry:" + this.entryPoint);
args.Add("/base:" + this.baseAddr);
args.Add("/PDB:" + this.getPdb());
return new ProcessInvokeAsyncWorker(
workingDirectory,
this,
linker,
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
getDiagnosticsBase(),
allowAbsoluteExe: true,
allowAbsoluteArgs: true);
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
List<string> args = new List<string>() { "/LARGEADDRESSAWARE", "/driver", "/fixed", "/subsystem:native", "/nodefaultlib" };
args.Add(this.objFile.getRelativePath());
args.Add("/out:" + this.outputObject.getRelativePath());
args.Add("/entry:" + this.entryPoint);
args.Add("/base:" + this.baseAddr);
return new ProcessInvokeAsyncWorker(
workingDirectory,
this,
this.getLinkerExe().getRelativePath(),
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
this.getDiagnosticsBase());
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
List<string> args = new List<string>() { "/Cp", "/c", "/Zd", "/Zf", "/Zi" };
args.Add("/Fo" + this.getObj().getRelativePath());
args.Add("/Fl" + this.getLis().getRelativePath());
////args.Add("/Fm" + getMap().getRelativePath());
// TODO: "/I$SPEC_INCLUDE_DIR"
args.Add(this.asmFile.getRelativePath());
return new ProcessInvokeAsyncWorker(
workingDirectory,
this,
this.getMasmExe().getRelativePath(),
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
this.getDiagnosticsBase());
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
BuildObject absDestPath = this.getAbsDestPath();
////Directory.Delete(vfs.ToPath(absDestPath), true); // This verb should be the only one writing here, so let's keep it tidy.
Directory.CreateDirectory(workingDirectory.PathTo(absDestPath)); // REVIEW: Shouldn't PrepareForVerb already do this?
string dafnyccExecutable = this.getExecutable().getRelativePath();
List<string> args = new List<string>();
args.AddRange(this.getRootArgs().Select<SourcePath, string>(sp => sp.getRelativePath()));
args.Add("/outdir:" + this.getDestPath());
this.addExtraArgs(args);
return new ProcessInvokeAsyncWorker(
workingDirectory,
this,
dafnyccExecutable,
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
getDiagnosticsBase());
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
List<string> args = new List<string>() { this.pythonScript.getRelativePath(), this.asmFileIn.getRelativePath() };
string python_exe = @"C:\Python27\pythonw.exe";
if (!File.Exists(python_exe))
{
throw new FileNotFoundException("Missing python executable: " + python_exe + ". Try installing from: https://www.python.org/");
}
return new ProcessInvokeAsyncWorker(
workingDirectory,
this,
python_exe,
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
getDiagnosticsBase(),
allowAbsoluteExe: true,
captureStdout: this.asmFileOut);
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
List <string> args = new List <string>()
{
"/Cp", "/c", "/Zd", "/Zf", "/Zi"
};
args.Add("/Fo" + this.getObj().getRelativePath());
args.Add("/Fl" + this.getLis().getRelativePath());
////args.Add("/Fm" + getMap().getRelativePath());
// TODO: "/I$SPEC_INCLUDE_DIR"
args.Add(this.asmFile.getRelativePath());
return(new ProcessInvokeAsyncWorker(
workingDirectory,
this,
this.getMasmExe().getRelativePath(),
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
this.getDiagnosticsBase()));
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
if (this.verificationRequest.isComplete())
{
VerificationResult vr = VerificationResult.fromXmlFile(this.batchSummaryVerb.getOutputFile());
if (!vr.pass)
{
Util.Assert(false); // Should never get here, since Ironclad app should fail before producing a verified exe.
return(new VerbSyncWorker(workingDirectory, new Failed()));
}
}
// Copy the AppLoader binary and the bootloader into the same directory as the app's binary, so the pxe-loader can find them.
// REVIEW: Not clear this is doing the right thing with shift to WorkingDirectory.
File.Copy(workingDirectory.PathTo(this.loaderVerb.getExe()), workingDirectory.PathTo(this.loaderCopy), true);
File.Copy(workingDirectory.PathTo(this.appVerb.getExe()), workingDirectory.PathTo(this.appExecutableCopy), true);
File.Copy(workingDirectory.PathTo(this.bootloader), workingDirectory.PathTo(this.bootloaderCopy), true);
this.writeBootFile(workingDirectory);
return(new VerbSyncWorker(workingDirectory, new Fresh()));
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
if (this.verificationRequest.isComplete())
{
// If the verification succeeded, then we convert the untrusted exe into a trusted exe (via a copy).
VerificationResult vr = VerificationResult.fromXmlFile(this.verifyResultsVerb.getOutputFile());
if (!vr.pass)
{
return(new VerbSyncWorker(workingDirectory, new Failed()));
}
File.Copy(workingDirectory.PathTo(this.srcObject), workingDirectory.PathTo(this.outputObject), true); // True => Overwrite
}
else
{
UnverifiedSentinelVirtualContents contents = new UnverifiedSentinelVirtualContents();
BuildEngine.theEngine.Repository.StoreVirtual(this.outputObject, new Fresh(), contents);
}
return(new VerbSyncWorker(workingDirectory, new Fresh()));
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
BuildObject absDestPath = this.getAbsDestPath();
////Directory.Delete(vfs.ToPath(absDestPath), true); // This verb should be the only one writing here, so let's keep it tidy.
Directory.CreateDirectory(workingDirectory.PathTo(absDestPath)); // REVIEW: Shouldn't PrepareForVerb already do this?
string dafnyccExecutable = this.getExecutable().getRelativePath();
List <string> args = new List <string>();
args.AddRange(this.getRootArgs().Select <SourcePath, string>(sp => sp.getRelativePath()));
args.Add("/outdir:" + this.getDestPath());
this.addExtraArgs(args);
return(new ProcessInvokeAsyncWorker(
workingDirectory,
this,
dafnyccExecutable,
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
getDiagnosticsBase()));
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
if (this.verificationRequest.isComplete())
{
VerificationResult vr = VerificationResult.fromXmlFile(this.batchSummaryVerb.getOutputFile());
if (!vr.pass)
{
Util.Assert(false); // Should never get here, since Ironclad app should fail before producing a verified exe.
return new VerbSyncWorker(workingDirectory, new Failed());
}
}
// Copy the AppLoader binary and the bootloader into the same directory as the app's binary, so the pxe-loader can find them.
// REVIEW: Not clear this is doing the right thing with shift to WorkingDirectory.
File.Copy(workingDirectory.PathTo(this.loaderVerb.getExe()), workingDirectory.PathTo(this.loaderCopy), true);
File.Copy(workingDirectory.PathTo(this.appVerb.getExe()), workingDirectory.PathTo(this.appExecutableCopy), true);
File.Copy(workingDirectory.PathTo(this.bootloader), workingDirectory.PathTo(this.bootloaderCopy), true);
this.writeBootFile(workingDirectory);
return new VerbSyncWorker(workingDirectory, new Fresh());
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
List<string> arguments = new List<string>();
arguments.Add("/noNLarith");
arguments.Add("/allowGlobals");
arguments.Add("/dafnycc");
arguments.Add("/z3opt:ARITH_RANDOM_SEED=1");
arguments.Add("/z3opt:NL_ARITH=false");
arguments.Add("/compile:0");
arguments.Add("/timeLimit:30");
arguments.Add("/noCheating:1");
foreach (string[] ann in new AnnotationScanner(this.dfysource).getAnnotations(ADDDAFNYFLAG_LABEL))
{
if (ann.Length != 2)
{
throw new SourceConfigurationError("Expected exactly 1 argument to " + ADDDAFNYFLAG_LABEL);
}
if (ann[1].StartsWith("/z3opt:NL_ARITH=true")) {
arguments.Remove("/noNLarith");
}
arguments.Add(ann[1]);
}
arguments.Add(this.dfysource.getRelativePath());
Logger.WriteLine("arguments: " + string.Join(" ", arguments));
return new ProcessInvokeAsyncWorker(
workingDirectory,
this,
DafnyExecutableDependencies.getDafnyExecutable().getRelativePath(),
arguments.ToArray(),
ProcessExitCodeHandling.NonzeroIsOkay,
getDiagnosticsBase(),
returnStandardOut: true,
returnStandardError: true,
allowCloudExecution: true);
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
List <string> arguments = new List <string>();
arguments.Add("/noNLarith");
arguments.Add("/allowGlobals");
arguments.Add("/z3opt:nlsat.randomize=false");
arguments.Add("/z3opt:pi.warnings=true");
arguments.Add("/proverWarnings:1");
arguments.Add("/compile:0");
arguments.Add("/timeLimit:30");
arguments.Add("/noCheating:1");
arguments.Add("/autoTriggers:1");
arguments.Add("/ironDafny");
foreach (string[] ann in new AnnotationScanner(this.dfysource).getAnnotations(ADDDAFNYFLAG_LABEL))
{
if (ann.Length != 2)
{
throw new SourceConfigurationError("Expected exactly 1 argument to " + ADDDAFNYFLAG_LABEL);
}
arguments.Add(ann[1]);
}
arguments.Add(this.dfysource.getRelativePath());
Logger.WriteLine("arguments: " + string.Join(" ", arguments));
return(new ProcessInvokeAsyncWorker(
workingDirectory,
this,
DafnyExecutableDependencies.getDafnyExecutable().getRelativePath(),
arguments.ToArray(),
ProcessExitCodeHandling.NonzeroIsOkay,
getDiagnosticsBase(),
returnStandardOut: true,
returnStandardError: true,
allowCloudExecution: true));
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
// Mimic this line from src\Checked\Nucleus\Main\build.ps1:
// _cat -out $OBJ\EntryCP_i.bpl -in $OBJ\MainCP_i.bpl,$SPEC_OBJ\EntryCP_i.bpl
// TODO: eliminate this special-case workaround.
try
{
// This is the trusted bit, creating the stitched ifc file.
////IEnumerable<string> ifcImports = extractImportStatements(dafnyMainIfcInput);
string[] mainLines = File.ReadAllLines(workingDirectory.PathTo(this.mainBeatVerb.getOutputs().First()));
string[] entryLines = File.ReadAllLines(workingDirectory.PathTo(this.genericStitch));
int sentinel_index = Array.IndexOf(entryLines, SENTINEL_APP_SPECIFIC_GOES_HERE);
if (sentinel_index < 0)
{
throw new UserError("Sentinel string missing in " + this.genericStitch);
}
IEnumerable<string> entryPrologue = entryLines.Take(sentinel_index + 1);
IEnumerable<string> entryEpilogue = entryLines.Skip(sentinel_index + 1);
string[] appSpecificLines = File.ReadAllLines(workingDirectory.PathTo(this.appSpecificStitch));
////File.WriteAllLines(getIfcOutput().getFilesystemPath(), ifcImports.Concat(mainLines.Concat(entryLines)));
File.WriteAllLines(
workingDirectory.PathTo(this.getIfcOutput()),
mainLines.Concat(entryPrologue).Concat(appSpecificLines).Concat(entryEpilogue));
// Here's some (at least untrusted) workaround, snarfing and repurposing the
// import list from dafny_Main_i up to Entry.imp.
IEnumerable<string> impImports = extractImportStatements(workingDirectory, this.dafnyMainImpInput);
string[] intext = File.ReadAllLines(workingDirectory.PathTo(this.entryImpInput));
File.WriteAllLines(workingDirectory.PathTo(this.getEntryImpOutput()), impImports.Concat(intext));
return new VerbSyncWorker(workingDirectory, new Fresh());
}
catch (IOException ex)
{
return new VerbSyncWorker(workingDirectory, new Failed(ex.ToString()));
}
}
public Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition)
{
if (disposition is Failed)
{
return(disposition);
}
HashSet <string> createdFiles = new HashSet <string>(Directory.GetFiles(workingDirectory.PathTo(this.getAbsDestPath())).Select(path => Path.GetFileName(path)));
HashSet <string> expectedFiles = new HashSet <string>(this.getOutputs().Select(obj => obj.getFileName()));
// DafnyCC/DafnySpec process a big batch of files together. Did we correctly understand what it did?
if (!createdFiles.SetEquals(expectedFiles))
{
// REVIEW: These are never used by anything. Remove?
bool dummy = createdFiles.SetEquals(expectedFiles);
int missing = expectedFiles.Except(createdFiles).Count();
int extra = createdFiles.Except(expectedFiles).Count();
string msg = "Missing files: " + string.Join(",", expectedFiles.Except(createdFiles)) + "\n" +
" Extra files: " + string.Join(",", createdFiles.Except(expectedFiles));
return(new Failed(msg));
}
// Propagate the NuBuild annotations.
foreach (InOutMapping mapping in this.getInOutMappings())
{
if (mapping.dfysource != null &&
this.transformFilterAccepts(mapping.dfysource))
{
AnnotationScanner.transferAnnotations(
workingDirectory, mapping.dfysource, mapping.basmIfc, BoogieAsmDepBase.CommentSymbol);
AnnotationScanner.transferAnnotations(
workingDirectory, mapping.dfysource, mapping.basmImp, BoogieAsmDepBase.CommentSymbol);
}
}
return(new Fresh());
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
// Mimic this line from src\Checked\Nucleus\Main\build.ps1:
// _cat -out $OBJ\EntryCP_i.bpl -in $OBJ\MainCP_i.bpl,$SPEC_OBJ\EntryCP_i.bpl
// TODO: eliminate this special-case workaround.
try
{
// This is the trusted bit, creating the stitched ifc file.
////IEnumerable<string> ifcImports = extractImportStatements(dafnyMainIfcInput);
string[] mainLines = File.ReadAllLines(workingDirectory.PathTo(this.mainBeatVerb.getOutputs().First()));
string[] entryLines = File.ReadAllLines(workingDirectory.PathTo(this.genericStitch));
int sentinel_index = Array.IndexOf(entryLines, SENTINEL_APP_SPECIFIC_GOES_HERE);
if (sentinel_index < 0)
{
throw new UserError("Sentinel string missing in " + this.genericStitch);
}
IEnumerable <string> entryPrologue = entryLines.Take(sentinel_index + 1);
IEnumerable <string> entryEpilogue = entryLines.Skip(sentinel_index + 1);
string[] appSpecificLines = File.ReadAllLines(workingDirectory.PathTo(this.appSpecificStitch));
////File.WriteAllLines(getIfcOutput().getFilesystemPath(), ifcImports.Concat(mainLines.Concat(entryLines)));
File.WriteAllLines(
workingDirectory.PathTo(this.getIfcOutput()),
mainLines.Concat(entryPrologue).Concat(appSpecificLines).Concat(entryEpilogue));
// Here's some (at least untrusted) workaround, snarfing and repurposing the
// import list from dafny_Main_i up to Entry.imp.
IEnumerable <string> impImports = extractImportStatements(workingDirectory, this.dafnyMainImpInput);
string[] intext = File.ReadAllLines(workingDirectory.PathTo(this.entryImpInput));
File.WriteAllLines(workingDirectory.PathTo(this.getEntryImpOutput()), impImports.Concat(intext));
return(new VerbSyncWorker(workingDirectory, new Fresh()));
}
catch (IOException ex)
{
return(new VerbSyncWorker(workingDirectory, new Failed(ex.ToString())));
}
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
// First expand our Dafny source file to inline all its includes.
this.expandedSource = this.input.getNewSourcePath("ExpandedSource.dfy");
DafnyIncludes dafnyIncludes = new DafnyIncludes();
dafnyIncludes.ExpandDafny(workingDirectory, this.input, this.expandedSource);
// Call Dafny.exe to compile Dafny source to CSharp target.
var args = new[] { "/noVerify", "/spillTargetCode:1", "/compile:2", "/ironDafny", this.expandedSource.getRelativePath() };
Console.WriteLine("expanded source: " + this.expandedSource.getRelativePath());
////Logger.WriteLine("arguments: " + String.Join(" ", args));
return
new ProcessInvokeAsyncWorker(
workingDirectory,
this,
DafnyExecutableDependencies.getDafnyExecutable().getRelativePath(),
args,
ProcessExitCodeHandling.NonzeroIsFailure,
getDiagnosticsBase(),
returnStandardOut: true, // REVIEW: Doesn't appear to be needed.
returnStandardError: true, // REVIEW: Doesn't appear to be needed.
allowCloudExecution: true);
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
List <string> args = new List <string>()
{
this.pythonScript.getRelativePath(), this.asmFileIn.getRelativePath()
};
string python_exe = @"C:\Python27\pythonw.exe";
if (!File.Exists(python_exe))
{
throw new FileNotFoundException("Missing python executable: " + python_exe + ". Try installing from: https://www.python.org/");
}
return(new ProcessInvokeAsyncWorker(
workingDirectory,
this,
python_exe,
args.ToArray(),
ProcessExitCodeHandling.NonzeroIsFailure,
getDiagnosticsBase(),
allowAbsoluteExe: true,
captureStdout: this.asmFileOut));
}
/// <summary>
/// Starts a task (i.e. runs a verb).
/// </summary>
/// <param name="verb">The verb to run.</param>
private void startTask(IVerb verb)
{
this.runningTasks += 1;
// We execute the verb in a private build tree (WorkingDirectory).
WorkingDirectory workingDirectory = new WorkingDirectory(BuildEngine.theEngine.getIronRoot());
// Note that we call PrepareForVerb prior to the verb's getWorker
// method as the getWorker call might do some of the work directly.
// REVIEW: We might want to change our contract with getWorker to
// disallow it from touching files in the working directory (so we
// don't have to prep the working dir in the remote execution case).
this.PrepareForVerb(workingDirectory, verb);
IVerbWorker worker = verb.getWorker(workingDirectory);
if (worker.IsSync() == VerbWorkerType.Sync)
{
this.completeTask(verb, worker);
}
else
{
AsyncVerbTask task = new AsyncVerbTask(this, worker, verb);
Say(string.Format("scheduling {0}", verb));
#pragma warning disable 162
if (DebugOneThread)
{
task.Run();
}
else
{
new Thread(new ThreadStart(task.Run)).Start();
}
#pragma warning restore 162
}
}
public override Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition)
{
// No cleanup to do.
return(disposition);
}
/// <summary>
/// Initializes a new instance of the TaskCompletion class.
/// </summary>
/// <param name="workingDirectory">
/// The private working directory the task executed in.
/// </param>
/// <param name="verb">
/// The verb whose completion this instance describes.
/// </param>
/// <param name="disposition">
/// The disposition of this task.
/// </param>
public TaskCompletion(WorkingDirectory workingDirectory, IVerb verb, Disposition disposition)
{
this.workingDirectory = workingDirectory;
this.verb = verb;
this.disposition = disposition;
}
public override Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition)
{
// No cleanup to do.
return disposition;
}
/// <summary>
/// Expand a dafny source file to include all of its includes inline.
/// </summary>
/// <param name="workingDirectory">Temporary working directory to use.</param>
/// <param name="input">Source build object.</param>
/// <param name="output">Where to create build object for expanded source file.</param>
public void ExpandDafny(WorkingDirectory workingDirectory, SourcePath input, SourcePath output)
{
// Prepare the output stream.
using (TextWriter outWriter = new StreamWriter(workingDirectory.PathTo(output)))
{
// Stash away a few things for use by our recursive helper function.
this.workingDirectory = workingDirectory;
this.outputWriter = outWriter;
this.visited = new HashSet<string>();
// Recursively expand the initial Dafny source file to inline all of its includes.
this.ExpandDafnyRecurse(input);
}
// Cache the output file in the Repository.
BuildEngine.theEngine.Repository.Store(workingDirectory, output, new Fresh());
}
public virtual void preprocess(WorkingDirectory workingDirectory)
{
}
/// <summary>
/// Record how each output of a task appeared.
/// </summary>
/// <param name="completion">The task completion notification.</param>
/// <returns>Overall result of the verb execution.</returns>
private Disposition recordResult(VerbRunner.TaskCompletion completion)
{
WorkingDirectory workingDirectory = completion.workingDirectory;
IVerb verb = completion.verb;
Disposition executionDisposition = completion.disposition;
List <BuildObjectValuePointer> outputs = new List <BuildObjectValuePointer>();
List <string> missingOutputs = new List <string>();
IEnumerable <BuildObject> expectedOutputs = verb.getOutputs();
if (executionDisposition is Failed)
{
expectedOutputs = expectedOutputs.Concat(verb.getFailureOutputs());
}
bool hasVirtualOutputs = false;
foreach (BuildObject outobj in expectedOutputs)
{
if (!(outobj is VirtualBuildObject))
{
// For expected file outputs, check for existence in working directory.
// REVIEW: Add method to WorkingDirectory which does this?
if (File.Exists(workingDirectory.PathTo(outobj)))
{
// Try to catch accidental case mismatches that would burn us when we
// try to fetch the file back in.
////string fsname = PathNormalizer.dbg_normalizePath_nocache(outobj.deprecatedGetFilesystemPath(), false);
////Util.Assert(Path.GetFileName(fsname).Equals(outobj.getFileName()));
// REVIEW: Do we need to worry about case mismatches anymore? See comments above.
outputs.Add(this.repository.Store(workingDirectory, outobj, executionDisposition));
// Store a copy of this verb output as a file in the real nuobj directory.
Util.Assert(outobj.getRelativePath().StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal));
string nuobjPath = IronRootDirectory.PathTo(outobj);
Directory.CreateDirectory(Path.GetDirectoryName(nuobjPath));
File.Copy(workingDirectory.PathTo(outobj), nuobjPath, true);
}
else
{
missingOutputs.Add(string.Format("Missing expected output {0}", outobj.getRelativePath()));
}
}
else
{
hasVirtualOutputs = true;
if (this.repository.GetDisposition(outobj) is Fresh)
{
// Nothing to cache; virtual objects only survive in the Repository, the in-process store.
}
else
{
missingOutputs.Add(string.Format("Missing expected virtual {0}", outobj.getRelativePath()));
}
}
}
if (!(executionDisposition is Failed) && missingOutputs.Count() > 0)
{
executionDisposition = new Failed(missingOutputs);
}
ResultSummaryRecord summary = new ResultSummaryRecord(verb, executionDisposition, outputs);
string inputHash = this.computeInputHash(verb, true);
Util.Assert(inputHash != null);
if (!hasVirtualOutputs)
{
this.repository.StoreResult(inputHash, summary);
}
else
{
this.Say("Not caching verb persistently: " + verb);
}
this.verbIsComplete(verb, executionDisposition);
return(executionDisposition);
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
// Read and aggregate all the input results.
int parseFailures = 0;
int verificationFailures = 0;
int timeouts = 0;
int filesWithParseFailures = 0;
int filesWithVerificationFailures = 0;
int filesWithTimeouts = 0;
int passCount = 0;
int failCount = 0;
double cpuTime = 0;
string failedFiles = "";
string timeoutFiles = "";
string parseFiles = "";
List <VerificationMessage> failMessages = new List <VerificationMessage>();
List <VerificationResult> results = new List <VerificationResult>();
// REVIEW: Why pull out the enumerator this way?
IEnumerable <BuildObject> verificationResultsEnumerator = this.verificationResults;
foreach (BuildObject verificationResult in verificationResultsEnumerator)
{
VerificationResult vr = VerificationResult.fromXmlFile(verificationResult);
results.Add(vr);
if (vr == null)
{
return(new VerbSyncWorker(
workingDirectory,
new Failed("Build system broke: missing VerificationResult for " + verificationResult)));
}
if (vr.pass)
{
passCount += 1;
}
else
{
failCount += 1;
failMessages.AddRange(vr.getMessages());
}
parseFailures += vr.parseFailures;
verificationFailures += vr.verificationFailures;
timeouts += vr.timeouts;
filesWithParseFailures += vr.parseFailures > 0 ? 1 : 0;
filesWithVerificationFailures += vr.verificationFailures > 0 ? 1 : 0;
filesWithTimeouts += vr.timeouts > 0 ? 1 : 0;
if (vr.verificationFailures > 0)
{
failedFiles = failedFiles + vr.sourceLabel + "\n";
}
if (vr.timeouts > 0)
{
timeoutFiles = timeoutFiles + vr.sourceLabel + "\n";
}
if (vr.parseFailures > 0)
{
parseFiles = parseFiles + vr.sourceLabel + "\n";
}
////Logger.WriteLine("Synthesizing cpuTime from " + verificationResult);
cpuTime += vr.cpuTime;
}
bool allPass = failCount == 0;
PresentationBuilder pr = new PresentationBuilder();
int any_failures = parseFailures + verificationFailures + timeouts;
string overall_status = any_failures > 0 ? "Fail" : "Success";
pr.pre(VerificationResult._VERIFICATION_RESULT_PLACEHOLDER + "\n");
pr.spacer();
pr.startHeader();
pr.color(this.colorByFailureCount(any_failures), "Overall status: " + overall_status);
pr.endHeader();
pr.line("Count of files with failures: " + failCount);
pr.startBullet();
pr.color(this.colorByFailureCount(filesWithParseFailures), "Files with parse failures: " + filesWithParseFailures.ToString());
pr.endBullet();
pr.startBullet();
pr.color(this.colorByFailureCount(filesWithVerificationFailures), "Files with verification failures: " + filesWithVerificationFailures.ToString());
pr.endBullet();
pr.startBullet();
pr.color(this.colorByFailureCount(filesWithVerificationFailures), "File Names with verification failures: " + failedFiles);
pr.endBullet();
pr.startBullet();
pr.color(this.colorByFailureCount(filesWithTimeouts), "Files with timeouts: " + filesWithTimeouts.ToString());
pr.endBullet();
pr.startBullet();
pr.color(this.colorByFailureCount(filesWithVerificationFailures), "File Names with timeout failures: " + timeoutFiles);
pr.endBullet();
pr.startBullet();
pr.color(this.colorByFailureCount(filesWithParseFailures), "File Names with parse failures: " + parseFiles);
pr.endBullet();
pr.spacer();
pr.header(string.Format("Total processing time: {0:0.0}s {1}", cpuTime, new TimeSpan((long)(cpuTime * 10000000L))));
int top_n = 10;
pr.header(string.Format("Slowest {0} verifications:", top_n));
results.Sort(this.ByCpuTimeDecreasing);
foreach (VerificationResult slowResult in results.Take(top_n))
{
pr.startBullet();
pr.color(
this.colorByFailureCount(slowResult.pass ? 0 : 1),
string.Format("{0,6:##0.0}s {1}", slowResult.cpuTime, slowResult.sourceLabel));
pr.endBullet();
}
foreach (VerificationMessage message in failMessages)
{
pr.spacer();
pr.header("Failure with " + message.SourceLabel);
pr.pre(message.Message);
}
Presentation pres = pr.fix();
VerificationResult outvr = new VerificationResult("summary", allPass, cpuTime, parseFailures, verificationFailures, timeouts, failMessages);
outvr.addXmlFiller(pres);
outvr.toXmlFile(workingDirectory.PathTo(this.outputObject));
this.setWasRejectableFailure(!outvr.pass);
return(new VerbSyncWorker(workingDirectory, new Fresh()));
}
protected override void postprocess(WorkingDirectory workingDirectory)
{
AnnotationScanner.transferAnnotations(workingDirectory, basmInput, outputFile(), BoogieVerb.CommentSymbol);
}
/// <summary>
/// Initializes a new instance of the CloudSubmitter class.
/// </summary>
/// <param name="requestIdentifier">
/// Unique identifier for this request.
/// </param>
/// <param name="workingDirectory">
/// Working directory the process is started in.
/// </param>
/// <param name="inputFiles">
/// List of input files expected by the process.
/// </param>
/// <param name="outputFiles">
/// List of potential output files generated by the process.
/// </param>
/// <param name="executable">Executable to run.</param>
/// <param name="args">
/// Command line arguments to provide to the executable.
/// </param>
/// <param name="failureBase">Not sure what this is -- some debugging/diagnostic thing.</param>
/// <param name="captureStdout">Where to (optionally) store the captured standard out.</param>
/// <param name="dbgText">Debugging text for something or another.</param>
public CloudSubmitter(
string requestIdentifier,
WorkingDirectory workingDirectory,
IEnumerable <BuildObject> inputFiles,
IEnumerable <BuildObject> outputFiles,
string executable,
string[] args,
BuildObject failureBase,
BuildObject captureStdout = null,
string dbgText = null)
{
// Catch bad verb authors before they hurt themselves.
Util.Assert(!executable.Contains(":")); // Hey, this looks like an absolute path! Use .getRelativePath(); it makes your output more stable.
foreach (string arg in args)
{
// Pardon my distasteful heuristic to avoid flagging /flag:value args.
Util.Assert(arg.Length < 2 || arg[1] != ':'); // Hey, this looks like an absolute path! Use .getRelativePath() to tolerate crossing machine boundaries.
}
// Stash away things we'll want to remember later.
this.workingDirectory = workingDirectory;
// Create list of input file mappings.
// REVIEW: We're not running on the main thread at this point. Are below calls all thread-safe?
List <BuildObjectValuePointer> inputFileMappings = new List <BuildObjectValuePointer>();
foreach (BuildObject file in inputFiles)
{
string fileHash = BuildEngine.theEngine.Repository.GetHash(file);
Util.Assert(!string.IsNullOrEmpty(fileHash));
inputFileMappings.Add(new BuildObjectValuePointer(fileHash, file.getRelativePath()));
// Ensure that the input files are in the cloud cache.
// REVIEW: best way to determine this is a source file?
ItemCacheContainer container;
if (file is SourcePath)
{
container = ItemCacheContainer.Sources;
}
else
{
container = ItemCacheContainer.Objects;
}
ItemCacheMultiplexer multiplexedCache = BuildEngine.theEngine.ItemCache as ItemCacheMultiplexer;
Util.Assert(multiplexedCache != null);
multiplexedCache.SyncItemToCloud(container, fileHash);
}
// Prepare cloud execution request for submission.
string arguments = string.Join(" ", args);
CloudExecutionRequest request = new CloudExecutionRequest(
BuildEngine.theEngine.CloudReportQueueName,
requestIdentifier,
CloudExecutionRequest.Operation.RunExecutable,
executable,
arguments,
inputFileMappings,
outputFiles);
BuildEngine.theEngine.CloudExecutionQueue.SubmitRequest(request);
// Wait for remote execution to finish.
int requestsOutstanding;
Console.WriteLine("Waiting on remote execution report for request '{0}'.", requestIdentifier);
CloudExecutionReport executionReport = BuildEngine.theEngine.CloudExecutionQueue.GetReport(requestIdentifier, out requestsOutstanding);
Console.WriteLine("Received remote execution report for request '{0}'. {1} others still outstanding.", requestIdentifier, requestsOutstanding);
// Record what we got back.
this.exitCode = executionReport.ExitCode;
this.stdout = executionReport.StandardOutput;
this.stderr = executionReport.StandardError;
this.cpuTime = executionReport.CpuTime;
// Copy output files from cloud cache back to local working dir.
// REVIEW: This is just to set things up as expected for the
// Scheduler's recordResult routine. Could re-architect this to
// be more efficient for the remote execution case.
foreach (BuildObjectValuePointer outputFileMapping in executionReport.OutputFileMappings)
{
BuildEngine.theEngine.CloudCache.FetchItemToFile(
ItemCacheContainer.Objects,
outputFileMapping.ObjectHash,
workingDirectory.PathTo(outputFileMapping.RelativePath));
}
}
// TODO: Rename obj to something meaningful. Is it a boot file?
private string mkBootFileEntry(WorkingDirectory workingDirectory, BuildObject obj)
{
return(string.Format("Size={0} Path=/{1}", new FileInfo(workingDirectory.PathTo(obj)).Length, obj.getFileName()));
}
/// <summary>
/// Initializes a new instance of the ProcessInvoker class.
/// </summary>
/// <param name="workingDirectory">The working directory the process is started in.</param>
/// <param name="executable">The executable to run.</param>
/// <param name="args">The command line arguments to provide to the executable.</param>
/// <param name="failureBase">Not sure what this is -- some debugging/diagnostic thing.</param>
/// <param name="captureStdout">Where to (optionally) capture standard out.</param>
/// <param name="dbgText">Debugging text for something or another.</param>
/// <param name="allowAbsoluteExe">Whether to allow an absolute (rather than relative) file path to the executable.</param>
/// <param name="allowAbsoluteArgs">Whether to allow absolute (rather than relative) file paths as arguments.</param>
/// <param name="workingDirOverride">The working directory to use.</param>
public ProcessInvoker(
WorkingDirectory workingDirectory,
string executable,
string[] args,
BuildObject failureBase,
BuildObject captureStdout = null,
string dbgText = null,
bool allowAbsoluteExe = false,
bool allowAbsoluteArgs = false,
string workingDirOverride = null)
{
// Catch bad verb authors before they hurt themselves.
Util.Assert(allowAbsoluteExe || !executable.Contains(":")); // Hey, this looks like an absolute path! Use .getRelativePath(); it makes your output more stable.
foreach (string arg in args)
{
// Pardon my distasteful heuristic to avoid flagging /flag:value args.
Util.Assert(allowAbsoluteArgs || arg.Length < 2 || arg[1] != ':'); // Hey, this looks like an absolute path! Use .getRelativePath() to tolerate crossing machine boundaries.
}
this.workingDirectory = workingDirectory;
this.stdout = new StringBuilder();
this.stderr = new StringBuilder();
using (Job job = new Job())
{
using (Process proc = new Process())
{
if (allowAbsoluteExe)
{
proc.StartInfo.FileName = executable;
}
else
{
// TODO: *All* async verbs need to list their executable (and all the libs it depends upon) as dependencies.
proc.StartInfo.FileName = workingDirectory.PathTo(executable);
}
// TODO Is there a better way to escape the args to avoid problems with spaces?
proc.StartInfo.Arguments = string.Join(" ", args);
proc.StartInfo.WorkingDirectory = workingDirOverride == null ? workingDirectory.Root : workingDirOverride;
proc.StartInfo.RedirectStandardOutput = true;
// REVIEW: Maybe we should always capture stdout in a StringBuilder and just write it out to a file afterwards if requested?
if (captureStdout != null)
{
this.tmpStdout = new BuildObject(captureStdout.getRelativePath() + ".tmp");
this.stdoutFile = new StreamWriter(workingDirectory.PathTo(this.tmpStdout));
proc.OutputDataReceived += new DataReceivedEventHandler(this.StdoutRedirectHandler);
}
else
{
// Collect stdout here for diagnostics.
proc.OutputDataReceived += new DataReceivedEventHandler(this.StdoutHandler);
}
proc.StartInfo.RedirectStandardError = true;
proc.ErrorDataReceived += new DataReceivedEventHandler(this.StderrHandler);
proc.StartInfo.UseShellExecute = false;
string commandLine = proc.StartInfo.FileName + " " + proc.StartInfo.Arguments;
if (failureBase != null && AlwaysEmitDiagnostics)
{
// In diagnostic mode, we emit the command line twice, once ahead in case Boogie decides
// to run away and never come back.
BuildObject failureBatObj = failureBase.makeOutputObject(".bat");
workingDirectory.CreateDirectoryFor(failureBatObj);
File.WriteAllText(workingDirectory.PathTo(failureBatObj), commandLine);
}
proc.Start();
job.AddProcess(proc);
proc.BeginOutputReadLine();
proc.BeginErrorReadLine();
proc.WaitForExit();
this.cpuTime = job.GetCpuTime().TotalSeconds;
this.exitCode = proc.ExitCode;
if (this.stdoutFile != null)
{
this.stdoutFile.Close();
}
if (failureBase != null && AlwaysEmitDiagnostics)
{
workingDirectory.CreateDirectoryFor(failureBase);
File.WriteAllText(workingDirectory.PathTo(failureBase.makeOutputObject(".bat")), commandLine);
File.WriteAllText(workingDirectory.PathTo(failureBase.makeOutputObject(".txt")), dbgText);
File.WriteAllText(workingDirectory.PathTo(failureBase.makeOutputObject(".stdout")), this.GetStdoutString());
File.WriteAllText(workingDirectory.PathTo(failureBase.makeOutputObject(".stderr")), this.GetStderr());
}
}
}
// REVIEW: Add Delete, Exists and Move methods to WorkingDirectory class?
if (this.tmpStdout != null && File.Exists(workingDirectory.PathTo(this.tmpStdout)))
{
// REVIEW: Nothing should be here. Bother with the delete?
File.Delete(workingDirectory.PathTo(captureStdout));
File.Move(workingDirectory.PathTo(this.tmpStdout), workingDirectory.PathTo(captureStdout));
this.tmpStdout = null;
}
}
// REVIEW: Make this private?
internal void injectAnnotations(WorkingDirectory workingDirectory, BuildObject dest, string commentToken)
{
string annotations = this.emit(commentToken);
// REVIEW: Improve upon this round-about way of prepending to a file?
string destStr = File.ReadAllText(workingDirectory.PathTo(dest));
File.Delete(workingDirectory.PathTo(dest));
File.WriteAllText(workingDirectory.PathTo(dest), annotations + destStr);
}
/// <summary>
/// Initializes a new instance of the TaskCompletion class.
/// </summary>
/// <param name="workingDirectory">
/// The private working directory the task executed in.
/// </param>
/// <param name="verb">
/// The verb whose completion this instance describes.
/// </param>
/// <param name="disposition">
/// The disposition of this task.
/// </param>
public TaskCompletion(WorkingDirectory workingDirectory, IVerb verb, Disposition disposition)
{
this.workingDirectory = workingDirectory;
this.verb = verb;
this.disposition = disposition;
}
public void store(WorkingDirectory workingDirectory, BuildObject location)
{
this.complete = true;
string[] lines = this.verificationObligations.Select(vo => vo.getRelativePath()).ToArray();
File.WriteAllLines(workingDirectory.PathTo(location), lines);
}
public override IVerbWorker getWorker(WorkingDirectory workingDirectory)
{
// Read and aggregate all the input results.
int parseFailures = 0;
int verificationFailures = 0;
int timeouts = 0;
int filesWithParseFailures = 0;
int filesWithVerificationFailures = 0;
int filesWithTimeouts = 0;
int passCount = 0;
int failCount = 0;
double cpuTime = 0;
List<VerificationMessage> failMessages = new List<VerificationMessage>();
List<VerificationResult> results = new List<VerificationResult>();
// REVIEW: Why pull out the enumerator this way?
IEnumerable<BuildObject> verificationResultsEnumerator = this.verificationResults;
foreach (BuildObject verificationResult in verificationResultsEnumerator)
{
VerificationResult vr = VerificationResult.fromXmlFile(verificationResult);
results.Add(vr);
if (vr == null)
{
return new VerbSyncWorker(
workingDirectory,
new Failed("Build system broke: missing VerificationResult for " + verificationResult));
}
if (vr.pass)
{
passCount += 1;
}
else
{
failCount += 1;
failMessages.AddRange(vr.getMessages());
}
parseFailures += vr.parseFailures;
verificationFailures += vr.verificationFailures;
timeouts += vr.timeouts;
filesWithParseFailures += vr.parseFailures > 0 ? 1 : 0;
filesWithVerificationFailures += vr.verificationFailures > 0 ? 1 : 0;
filesWithTimeouts += vr.timeouts > 0 ? 1 : 0;
////Logger.WriteLine("Synthesizing cpuTime from " + verificationResult);
cpuTime += vr.cpuTime;
}
bool allPass = failCount == 0;
PresentationBuilder pr = new PresentationBuilder();
int any_failures = parseFailures + verificationFailures + timeouts;
string overall_status = any_failures > 0 ? "Fail" : "Success";
pr.pre(VerificationResult._VERIFICATION_RESULT_PLACEHOLDER+"\n");
pr.spacer();
pr.startHeader();
pr.color(this.colorByFailureCount(any_failures), "Overall status: " + overall_status);
pr.endHeader();
pr.line("Count of files with failures: " + failCount);
pr.startBullet();
pr.color(this.colorByFailureCount(filesWithParseFailures), "Files with parse failures: " + filesWithParseFailures.ToString());
pr.endBullet();
pr.startBullet();
pr.color(this.colorByFailureCount(filesWithVerificationFailures), "Files with verification failures: " + filesWithVerificationFailures.ToString());
pr.endBullet();
pr.startBullet();
pr.color(this.colorByFailureCount(filesWithTimeouts), "Files with timeouts: " + filesWithTimeouts.ToString());
pr.endBullet();
pr.spacer();
pr.header(string.Format("Total processing time: {0:0.0}s {1}", cpuTime, new TimeSpan((long)(cpuTime * 10000000L))));
int top_n = 10;
pr.header(string.Format("Slowest {0} verifications:", top_n));
results.Sort(this.ByCpuTimeDecreasing);
foreach (VerificationResult slowResult in results.Take(top_n))
{
pr.startBullet();
pr.color(
this.colorByFailureCount(slowResult.pass ? 0 : 1),
string.Format("{0,6:##0.0}s {1}", slowResult.cpuTime, slowResult.sourceLabel));
pr.endBullet();
}
foreach (VerificationMessage message in failMessages)
{
pr.spacer();
pr.header("Failure with " + message.SourceLabel);
pr.pre(message.Message);
}
Presentation pres = pr.fix();
VerificationResult outvr = new VerificationResult("summary", allPass, cpuTime, parseFailures, verificationFailures, timeouts, failMessages);
outvr.addXmlFiller(pres);
outvr.toXmlFile(workingDirectory.PathTo(this.outputObject));
this.setWasRejectableFailure(!outvr.pass);
return new VerbSyncWorker(workingDirectory, new Fresh());
}
/// <summary>
/// Run the requested executable and produce a report of the results.
/// </summary>
/// <param name="executionRequest">The execution request.</param>
/// <returns>A report of the results.</returns>
private CloudExecutionReport RunAnExecutable(CloudExecutionRequest executionRequest)
{
// REVIEW: How/whether to use this.
BuildObject diagnosticsBase = new BuildObject(Path.Combine("nuobj", "diagnostics", "process"));
// Prep working directory with input files and output dirs.
// TODO: The below will throw cache exceptions if something
// isn't there (they should all be though). Need to catch
// these and fail the execution request when this happens.
WorkingDirectory workingDirectory = new WorkingDirectory(this.virtualIronRoot);
foreach (BuildObjectValuePointer inputFile in executionRequest.InputFileMappings)
{
// REVIEW: How to determine cache container here.
ItemCacheContainer container = ItemCacheContainer.Sources;
if (this.multiplexedItemCache.GetItemSize(container, inputFile.ObjectHash) == -1)
{
container = ItemCacheContainer.Objects;
}
// TODO: Move path/directory manipulation code into
// WorkingDirectory and/or ItemCache.
string inputFilePath = workingDirectory.PathTo(inputFile.RelativePath);
Directory.CreateDirectory(Path.GetDirectoryName(inputFilePath)); // REVIEW: Still neeeded?
this.multiplexedItemCache.FetchItemToFile(
container,
inputFile.ObjectHash,
inputFilePath);
}
foreach (BuildObject outputFile in executionRequest.OutputFiles)
{
workingDirectory.CreateDirectoryFor(outputFile);
}
// Run executable.
ProcessInvoker pinv = new ProcessInvoker(
workingDirectory,
executionRequest.Executable,
new string[] { executionRequest.Arguments },
diagnosticsBase,
null, // This is captureStdout. TODO: Should cleanup how this is used in ProcessInvoker.
null); // This is dbgText. REVIEW: How/whether to use this.
// When ProcessInvoker's constructor returns, the process has
// finished running.
Console.WriteLine("Request {0} completed in {1} seconds.", executionRequest.Identifier, pinv.CpuTime);
// Store output files in the (cloud) item cache, and create a
// list of the mappings.
List<BuildObjectValuePointer> outputFileMappings = new List<BuildObjectValuePointer>();
foreach (BuildObject outFile in executionRequest.OutputFiles)
{
if (File.Exists(workingDirectory.PathTo(outFile)))
{
string fileHash = Util.hashFilesystemPath(workingDirectory.PathTo(outFile));
Util.Assert(!string.IsNullOrEmpty(fileHash));
// Note we explicitly write to the cloud cache here.
this.cloudCache.StoreItemFromFile(ItemCacheContainer.Objects, fileHash, workingDirectory.PathTo(outFile));
outputFileMappings.Add(new BuildObjectValuePointer(fileHash, outFile.getRelativePath()));
}
}
// Collect the results into a report.
CloudExecutionReport report = new CloudExecutionReport(
executionRequest.Identifier,
CloudExecutionReport.StatusCode.Completed,
pinv.ExitCode,
pinv.GetStdout(),
pinv.GetStderr(),
pinv.CpuTime,
outputFileMappings);
return report;
}
public void store(WorkingDirectory workingDirectory, BuildObject location)
{
this.complete = true;
string[] lines = this.verificationObligations.Select(vo => vo.getRelativePath()).ToArray();
File.WriteAllLines(workingDirectory.PathTo(location), lines);
}
public virtual void preprocess(WorkingDirectory workingDirectory) { }
protected virtual void postprocess(WorkingDirectory workingDirectory)
{
}
public abstract IVerbWorker getWorker(WorkingDirectory workingDirectory);
public Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition)
{
return(disposition);
}
public virtual Disposition Complete(WorkingDirectory workingDirectory, double cpuTimeSeconds, string stdout, string stderr, Disposition disposition)
{
return disposition;
}
internal static void transferAnnotations(WorkingDirectory workingDirectory, BuildObject source, BuildObject dest, string commentToken)
{
new AnnotationScanner(source).injectAnnotations(workingDirectory, dest, commentToken);
}
public override void preprocess(WorkingDirectory workingDirectory)
{
base.preprocess(workingDirectory);
File.Copy(workingDirectory.PathTo(basmIn), workingDirectory.PathTo(getTmpInputFile()), true);
}
internal static void transferAnnotations(WorkingDirectory workingDirectory, BuildObject source, BuildObject dest, string commentToken)
{
new AnnotationScanner(source).injectAnnotations(workingDirectory, dest, commentToken);
}
/// <summary>
/// Starts a task (i.e. runs a verb).
/// </summary>
/// <param name="verb">The verb to run.</param>
private void startTask(IVerb verb)
{
this.runningTasks += 1;
// We execute the verb in a private build tree (WorkingDirectory).
WorkingDirectory workingDirectory = new WorkingDirectory(BuildEngine.theEngine.getIronRoot());
// Note that we call PrepareForVerb prior to the verb's getWorker
// method as the getWorker call might do some of the work directly.
// REVIEW: We might want to change our contract with getWorker to
// disallow it from touching files in the working directory (so we
// don't have to prep the working dir in the remote execution case).
this.PrepareForVerb(workingDirectory, verb);
IVerbWorker worker = verb.getWorker(workingDirectory);
if (worker.IsSync() == VerbWorkerType.Sync)
{
this.completeTask(verb, worker);
}
else
{
AsyncVerbTask task = new AsyncVerbTask(this, worker, verb);
Say(string.Format("scheduling {0}", verb));
#pragma warning disable 162
if (DebugOneThread)
{
task.Run();
}
else
{
new Thread(new ThreadStart(task.Run)).Start();
}
#pragma warning restore 162
}
}
