/// <summary>
/// Mark a verb as completed, and adjust our schedule accordingly.
/// </summary>
/// <param name="verb">The verb that completed.</param>
/// <param name="disp">The disposition of the verb's execution.</param>
private void verbIsComplete(IVerb verb, Disposition disp)
{
////Say(string.Format(" {0} is complete: {1}", verb, dbgDisposition));
////if (disp is Failed)
////{
//// // Failures can be hard to debug, since they don't leave any
//// // output in nuobj/. So report these even if they aren't
//// // built this run.
//// emitRealtimeReport(verb, disp);
////}
// Invariant: all of this verb's objs are non-Stale.
foreach (BuildObject obj in verb.getOutputs())
{
////Say(string.Format(" waking {0}", obj));
IEnumerable <IVerb> wokenSet = this.waitIndex.awaken(obj);
////foreach (IVerb wokenVerb in wokenSet)
////{
//// //Say(string.Format(" {0} woken", wokenVerb));
////}
this.nextVerbs.UnionWith(wokenSet);
}
this.emitRealtimeReport(verb, disp);
this.requiredVerbs.Remove(verb);
this.completedVerbs.Add(verb);
}
/// <summary>
/// Adds a verb to the mapping of build objects to the verb which
/// creates them.
/// </summary>
/// <remarks>
/// TODO: Make this private.
/// </remarks>
/// <param name="verb">The verb to add.</param>
internal void addVerb(IVerb verb)
{
if (!this.knownVerbs.Add(verb))
{
// We've already added this verb.
return;
}
// Add all verb outputs to the output-to-verb map.
foreach (BuildObject obj in verb.getOutputs())
{
if (this.outputToVerbMap.ContainsKey(obj))
{
Util.Assert(this.outputToVerbMap[obj].Equals(verb));
}
else
{
this.outputToVerbMap[obj] = verb;
}
}
// Recursively add all the verbs this verb is dependent upon,
// so that we have a complete index of outputs back to the
// verbs that generate them.
foreach (IVerb dependentVerb in verb.getVerbs())
{
this.addVerb(dependentVerb);
}
}
void verbIsComplete(IVerb verb, Disposition disp, BlessingRequest blessingRequest)
{
emitRealtimeReport(verb, disp);
//-Say(String.Format(" {0} is complete: {1}", verb, dbgDisposition));
//-if (disp is Failed)
//-{
//-
//-
//-
//- emitRealtimeReport(verb, disp);
//-}
//- invariant: all of this verb's objs are non-Stale.
foreach (BuildObject obj in verb.getOutputs())
{
if (blessingRequest == BlessingRequest.Bless)
{
nuObjContents.blessGeneral(obj, disp);
}
//-Say(String.Format(" waking {0}", obj));
IEnumerable <IVerb> wokenSet = waitIndex.awaken(obj);
//-foreach (IVerb wokenVerb in wokenSet)
//-{
//-
//-}
nextVerbs.UnionWith(wokenSet);
}
requiredVerbs.Remove(verb);
completedVerbs.Add(verb);
}
void prepareObjDirectory(IVerb verb)
{
foreach (BuildObject obj in verb.getOutputs())
{
obj.prepareObjDirectory();
}
}
/// <summary>
/// Prepares the working directory tree for a verb's execution.
/// </summary>
/// <param name="verb">The verb whose execution we're preparing for.</param>
private void PrepareForVerb(WorkingDirectory workingDirectory, IVerb verb)
{
// Debugging aide: write out the abstract id for this verb.
File.WriteAllText(workingDirectory.PathTo("Debug.txt"), verb.getAbstractIdentifier().ToString());
Repository repository = BuildEngine.theEngine.Repository;
// Copy all verb inputs from the item cache to here.
DependencyDisposition ddisp;
foreach (BuildObject input in verb.getDependencies(out ddisp))
{
if (!(input is VirtualBuildObject))
{
workingDirectory.CreateDirectoryFor(input); // REVIEW: No longer needed?
repository.Fetch(workingDirectory, input);
}
}
// Ensures that the directory tree for each of the verb's outputs exists.
foreach (BuildObject output in verb.getOutputs())
{
workingDirectory.CreateDirectoryFor(output);
}
}
public override IEnumerable <BuildObject> getDependencies(out DependencyDisposition ddisp)
{
ddisp = DependencyDisposition.Complete;
//- I really don't care how many outputs the parent has; any one will
//- link me to the paarent.
IEnumerable <BuildObject> result = _parent.getOutputs();
Util.Assert(result.Count() > 0);
return(result);
}
void IHasher.addVerb(IVerb verb)
{
foreach (BuildObject obj in verb.getOutputs())
{
if (outputToVerbMap.ContainsKey(obj))
{
Util.Assert(outputToVerbMap[obj].Equals(verb));
}
else
{
outputToVerbMap[obj] = verb;
}
}
}
//- Contract: call only when output objects are known to be cached
//- (because fetchResult returned non-Stale).
public void fetchOutputObjects(IVerb verb, IEnumerable <BuildObjectValuePointer> values, Disposition disp)
{
if (this.alreadyFetchedVerbs.Contains(verb))
{
return;
}
IEnumerable <BuildObject> objects = verb.getOutputs();
IEnumerable <BuildObject> failureObjs = verb.getFailureOutputs();
objects = objects.Concat(failureObjs);
Dictionary <string, BuildObject> objectDict = new Dictionary <string, BuildObject>();
foreach (BuildObject obj in objects)
{
objectDict.Add(obj.getRelativePath(), obj);
}
HashSet <BuildObject> recorded = new HashSet <BuildObject>();
foreach (BuildObjectValuePointer value in values)
{
if (objectDict.ContainsKey(value.relativePath))
{
BuildObject obj = objectDict[value.relativePath];
obj.prepareObjDirectory();
this.fetchObject(value, obj);
nuObjectContents.bless(obj, value.objectHash, disp);
recorded.Add(obj);
}
else
{
throw new Exception("Distressing: some BOVPs aren't in obj.getOutputs");
}
}
IEnumerable <BuildObject> unrecorded = objects.Except(recorded).Except(failureObjs);
Util.Assert(unrecorded.Count() == 0 || disp is Failed);
foreach (BuildObject obj in unrecorded)
{
nuObjectContents.bless(obj, null, disp);
}
this.alreadyFetchedVerbs.Add(verb);
}
/// <summary>
/// Mark a verb as having failed.
/// </summary>
/// <param name="verb">The verb that failed.</param>
private void markFailed(IVerb verb)
{
// At least one of verb's inputs has a permanent failure, so we didn't
// even try to execute it.
Disposition disposition = new Failed("upstream failure");
ResultSummaryRecord summary = new ResultSummaryRecord(verb, disposition, new BuildObjectValuePointer[] { });
// NB never store upstream failures to the persistent cache, because
// they depend on our knowledge this run that the upstream verb failed.
// If, in another run, the verb or its inputs are modified, produce the
// same outputs, but returns success, we'll be stuck pulling this
// upstream failure out of cache but calling this verb a failure.
////string inputHash = computeInputHash(verb, false);
////if (inputHash != null)
////{
//// Util.Assert(false);
//// // "Upstream failures" will never have a computable inputHash, because their inputs
//// // can't be considered known. Even if the upstream verb wrote something to disk,
//// // what if the upstream verb changes to no longer fail but still emit the same thing?
//// // We wouldn't want to conclude that, because the inputs hadn't changed, this
//// // verb still had an upstream failure.
//// repository.StoreResult(inputHash, summary);
////}
////else
////{
this.unrecordableFailures[verb] = disposition;
////}
// Mark all the verb's outputs as Failed in the repository.
foreach (BuildObject obj in verb.getOutputs())
{
if (obj is VirtualBuildObject)
{
this.repository.StoreVirtual(obj, disposition, null);
}
else
{
this.repository.AddObject(obj, disposition, null);
}
}
this.verbIsComplete(verb, disposition);
}
/// <summary>
/// Adds a verb to the mapping of build objects to the verb which
/// creates them.
/// </summary>
/// <remarks>
/// TODO: Make this private.
/// </remarks>
/// <param name="verb">The verb to add.</param>
internal void addVerb(IVerb verb)
{
if (!this.knownVerbs.Add(verb))
{
// We've already added this verb.
return;
}
// Add all verb outputs to the output-to-verb map.
foreach (BuildObject obj in verb.getOutputs())
{
if (this.outputToVerbMap.ContainsKey(obj))
{
Util.Assert(this.outputToVerbMap[obj].Equals(verb));
}
else
{
this.outputToVerbMap[obj] = verb;
}
}
// Recursively add all the verbs this verb is dependent upon,
// so that we have a complete index of outputs back to the
// verbs that generate them.
foreach (IVerb dependentVerb in verb.getVerbs())
{
this.addVerb(dependentVerb);
}
}
Disposition recordResult(IVerb verb, Disposition executionDisposition)
{
//- record how each output appeared.
List <BuildObjectValuePointer> outputs = new List <BuildObjectValuePointer>();
List <string> missingOutputs = new List <string>();
IEnumerable <BuildObject> expectedOutputs = verb.getOutputs();
if (executionDisposition is Failed)
{
expectedOutputs = expectedOutputs.Concat(verb.getFailureOutputs());
}
bool hasVirtualOutputs = false;
foreach (BuildObject outobj in expectedOutputs)
{
if (!(outobj is VirtualBuildObject))
{
if (File.Exists(outobj.getFilesystemPath()))
{
//- Try to catch accidental case mismatches that would burn us when we
//- try to fetch the file back in.
string fsname = PathNormalizer.dbg_normalizePath_nocache(outobj.getFilesystemPath(), false);
Util.Assert(Path.GetFileName(fsname).Equals(outobj.getFileName()));
outputs.Add(resultCache.storeObject(outobj));
}
else
{
missingOutputs.Add(String.Format("Missing expected output {0}", outobj.getRelativePath()));
}
}
else
{
hasVirtualOutputs = true;
if (nuObjContents.getDisposition(outobj) is Fresh)
{
//- nothing to cache; virtual objects only survive in nuObjContents, the in-process cache
}
else
{
missingOutputs.Add(String.Format("Missing expected virtual {0}", outobj.getRelativePath()));
}
}
}
if (!(executionDisposition is Failed) && missingOutputs.Count() > 0)
{
executionDisposition = new Failed(missingOutputs);
}
ResultSummaryRecord summary = new ResultSummaryRecord(verb, executionDisposition, outputs);
string inputHash = computeInputHash(verb, true);
Util.Assert(inputHash != null);
if (!hasVirtualOutputs)
{
resultCache.storeResult(inputHash, summary);
}
else
{
Say("Not caching verb persistently: " + verb);
}
verbIsComplete(verb, executionDisposition, BlessingRequest.Bless);
return(executionDisposition);
}
/// <summary>
/// Prepares the working directory tree for a verb's execution.
/// </summary>
/// <param name="verb">The verb whose execution we're preparing for.</param>
private void PrepareForVerb(WorkingDirectory workingDirectory, IVerb verb)
{
// Debugging aide: write out the abstract id for this verb.
File.WriteAllText(workingDirectory.PathTo("Debug.txt"), verb.getAbstractIdentifier().ToString());
Repository repository = BuildEngine.theEngine.Repository;
// Copy all verb inputs from the item cache to here.
DependencyDisposition ddisp;
foreach (BuildObject input in verb.getDependencies(out ddisp))
{
if (!(input is VirtualBuildObject))
{
workingDirectory.CreateDirectoryFor(input); // REVIEW: No longer needed?
repository.Fetch(workingDirectory, input);
}
}
// Ensures that the directory tree for each of the verb's outputs exists.
foreach (BuildObject output in verb.getOutputs())
{
workingDirectory.CreateDirectoryFor(output);
}
}
/// <summary>
/// Adds the output objects from a cached verb execution to the
/// repository, and ensures they are present in the item cache.
/// </summary>
/// <param name="verb">The verb whose outputs to add.</param>
/// <param name="resultRecord">
/// The result summary record of the verb execution.
/// </param>
/// <remarks>
/// Call only when output objects are known to be cached
/// (i.e. because FetchResult returned non-Stale).
/// REVIEW: This function probably shouldn't be in this file.
/// It does something similar for cached verb results that
/// the scheduler's recordResult method does for new verb
/// executions. Move this there and/or refactor?
/// </remarks>
public void AddVerbResults(IVerb verb, ResultSummaryRecord resultRecord)
{
if (this.alreadyAddedVerbs.Contains(verb))
{
// We only need to add a cached verb execution's outputs once.
return;
}
Disposition disposition = resultRecord.Disposition;
// REVIEW: In the below, some of these IEnumerables should be
// HashSets, and the HashSet should be a simple List.
// Create a collection of the potential outputs.
IEnumerable <BuildObject> outputs = verb.getOutputs();
IEnumerable <BuildObject> failureOutputs = verb.getFailureOutputs();
outputs = outputs.Concat(failureOutputs);
Dictionary <string, BuildObject> potentialOutputs = new Dictionary <string, BuildObject>();
foreach (BuildObject obj in outputs)
{
potentialOutputs.Add(obj.getRelativePath(), obj);
}
// Compare the actual outputs with the potential outputs,
// and add the actual ones to the repository.
HashSet <BuildObject> recorded = new HashSet <BuildObject>();
foreach (BuildObjectValuePointer actualOutput in resultRecord.Outputs)
{
if (potentialOutputs.ContainsKey(actualOutput.RelativePath))
{
BuildObject obj = potentialOutputs[actualOutput.RelativePath];
// TODO: Verify that the object exists in the item cache!
this.AddObject(obj, disposition, actualOutput.ObjectHash);
recorded.Add(obj);
// Store a copy of this verb output as a file in the real nuobj directory.
Util.Assert(actualOutput.RelativePath.StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal));
this.itemCache.FetchItemToFile(ItemCacheContainer.Objects, actualOutput.ObjectHash, IronRootDirectory.PathTo(actualOutput.RelativePath));
}
else
{
// Complain if we find interloping outputs.
throw new Exception("Distressing: some actual verb outputs aren't in the verb's list of potential outputs");
}
}
// Create a collection of missing outputs.
IEnumerable <BuildObject> unrecorded = outputs.Except(recorded).Except(failureOutputs);
// For non-Failed verb runs, complain if all expected outputs don't
// show up in the actual outputs.
Util.Assert(unrecorded.Count() == 0 || disposition is Failed);
// For cached verb runs with permanent failures (i.e. disposition
// is Failed), we want to mark all of the expected outputs as Failed
// even if no corresponding actual output was produced during the
// failed verb run.
foreach (BuildObject obj in unrecorded)
{
this.AddObject(obj, disposition, null);
}
// Remember that we've already added this verb's outputs.
this.alreadyAddedVerbs.Add(verb);
}
/// <summary>
/// Adds the output objects from a cached verb execution to the
/// repository, and ensures they are present in the item cache.
/// </summary>
/// <param name="verb">The verb whose outputs to add.</param>
/// <param name="resultRecord">
/// The result summary record of the verb execution.
/// </param>
/// <remarks>
/// Call only when output objects are known to be cached
/// (i.e. because FetchResult returned non-Stale).
/// REVIEW: This function probably shouldn't be in this file.
/// It does something similar for cached verb results that
/// the scheduler's recordResult method does for new verb
/// executions. Move this there and/or refactor?
/// </remarks>
public void AddVerbResults(IVerb verb, ResultSummaryRecord resultRecord)
{
if (this.alreadyAddedVerbs.Contains(verb))
{
// We only need to add a cached verb execution's outputs once.
return;
}
Disposition disposition = resultRecord.Disposition;
// REVIEW: In the below, some of these IEnumerables should be
// HashSets, and the HashSet should be a simple List.
// Create a collection of the potential outputs.
IEnumerable<BuildObject> outputs = verb.getOutputs();
IEnumerable<BuildObject> failureOutputs = verb.getFailureOutputs();
outputs = outputs.Concat(failureOutputs);
Dictionary<string, BuildObject> potentialOutputs = new Dictionary<string, BuildObject>();
foreach (BuildObject obj in outputs)
{
potentialOutputs.Add(obj.getRelativePath(), obj);
}
// Compare the actual outputs with the potential outputs,
// and add the actual ones to the repository.
HashSet<BuildObject> recorded = new HashSet<BuildObject>();
foreach (BuildObjectValuePointer actualOutput in resultRecord.Outputs)
{
if (potentialOutputs.ContainsKey(actualOutput.RelativePath))
{
BuildObject obj = potentialOutputs[actualOutput.RelativePath];
// TODO: Verify that the object exists in the item cache!
this.AddObject(obj, disposition, actualOutput.ObjectHash);
recorded.Add(obj);
// Store a copy of this verb output as a file in the real nuobj directory.
Util.Assert(actualOutput.RelativePath.StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal));
this.itemCache.FetchItemToFile(ItemCacheContainer.Objects, actualOutput.ObjectHash, IronRootDirectory.PathTo(actualOutput.RelativePath));
}
else
{
// Complain if we find interloping outputs.
throw new Exception("Distressing: some actual verb outputs aren't in the verb's list of potential outputs");
}
}
// Create a collection of missing outputs.
IEnumerable<BuildObject> unrecorded = outputs.Except(recorded).Except(failureOutputs);
// For non-Failed verb runs, complain if all expected outputs don't
// show up in the actual outputs.
Util.Assert(unrecorded.Count() == 0 || disposition is Failed);
// For cached verb runs with permanent failures (i.e. disposition
// is Failed), we want to mark all of the expected outputs as Failed
// even if no corresponding actual output was produced during the
// failed verb run.
foreach (BuildObject obj in unrecorded)
{
this.AddObject(obj, disposition, null);
}
// Remember that we've already added this verb's outputs.
this.alreadyAddedVerbs.Add(verb);
}
/// <summary>
/// Mark a verb as having failed.
/// </summary>
/// <param name="verb">The verb that failed.</param>
private void markFailed(IVerb verb)
{
// At least one of verb's inputs has a permanent failure, so we didn't
// even try to execute it.
Disposition disposition = new Failed("upstream failure");
ResultSummaryRecord summary = new ResultSummaryRecord(verb, disposition, new BuildObjectValuePointer[] { });
// NB never store upstream failures to the persistent cache, because
// they depend on our knowledge this run that the upstream verb failed.
// If, in another run, the verb or its inputs are modified, produce the
// same outputs, but returns success, we'll be stuck pulling this
// upstream failure out of cache but calling this verb a failure.
////string inputHash = computeInputHash(verb, false);
////if (inputHash != null)
////{
//// Util.Assert(false);
//// // "Upstream failures" will never have a computable inputHash, because their inputs
//// // can't be considered known. Even if the upstream verb wrote something to disk,
//// // what if the upstream verb changes to no longer fail but still emit the same thing?
//// // We wouldn't want to conclude that, because the inputs hadn't changed, this
//// // verb still had an upstream failure.
//// repository.StoreResult(inputHash, summary);
////}
////else
////{
this.unrecordableFailures[verb] = disposition;
////}
// Mark all the verb's outputs as Failed in the repository.
foreach (BuildObject obj in verb.getOutputs())
{
if (obj is VirtualBuildObject)
{
this.repository.StoreVirtual(obj, disposition, null);
}
else
{
this.repository.AddObject(obj, disposition, null);
}
}
this.verbIsComplete(verb, disposition);
}
/// <summary>
/// Mark a verb as completed, and adjust our schedule accordingly.
/// </summary>
/// <param name="verb">The verb that completed.</param>
/// <param name="disp">The disposition of the verb's execution.</param>
private void verbIsComplete(IVerb verb, Disposition disp)
{
////Say(string.Format(" {0} is complete: {1}", verb, dbgDisposition));
////if (disp is Failed)
////{
//// // Failures can be hard to debug, since they don't leave any
//// // output in nuobj/. So report these even if they aren't
//// // built this run.
//// emitRealtimeReport(verb, disp);
////}
// Invariant: all of this verb's objs are non-Stale.
foreach (BuildObject obj in verb.getOutputs())
{
////Say(string.Format(" waking {0}", obj));
IEnumerable<IVerb> wokenSet = this.waitIndex.awaken(obj);
////foreach (IVerb wokenVerb in wokenSet)
////{
//// //Say(string.Format(" {0} woken", wokenVerb));
////}
this.nextVerbs.UnionWith(wokenSet);
}
this.emitRealtimeReport(verb, disp);
this.requiredVerbs.Remove(verb);
this.completedVerbs.Add(verb);
}
/// <summary>
/// Record how each output of a task appeared.
/// </summary>
/// <param name="completion">The task completion notification.</param>
/// <returns>Overall result of the verb execution.</returns>
private Disposition recordResult(VerbRunner.TaskCompletion completion)
{
WorkingDirectory workingDirectory = completion.workingDirectory;
IVerb verb = completion.verb;
Disposition executionDisposition = completion.disposition;
List <BuildObjectValuePointer> outputs = new List <BuildObjectValuePointer>();
List <string> missingOutputs = new List <string>();
IEnumerable <BuildObject> expectedOutputs = verb.getOutputs();
if (executionDisposition is Failed)
{
expectedOutputs = expectedOutputs.Concat(verb.getFailureOutputs());
}
bool hasVirtualOutputs = false;
foreach (BuildObject outobj in expectedOutputs)
{
if (!(outobj is VirtualBuildObject))
{
// For expected file outputs, check for existence in working directory.
// REVIEW: Add method to WorkingDirectory which does this?
if (File.Exists(workingDirectory.PathTo(outobj)))
{
// Try to catch accidental case mismatches that would burn us when we
// try to fetch the file back in.
////string fsname = PathNormalizer.dbg_normalizePath_nocache(outobj.deprecatedGetFilesystemPath(), false);
////Util.Assert(Path.GetFileName(fsname).Equals(outobj.getFileName()));
// REVIEW: Do we need to worry about case mismatches anymore? See comments above.
outputs.Add(this.repository.Store(workingDirectory, outobj, executionDisposition));
// Store a copy of this verb output as a file in the real nuobj directory.
Util.Assert(outobj.getRelativePath().StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal));
string nuobjPath = IronRootDirectory.PathTo(outobj);
Directory.CreateDirectory(Path.GetDirectoryName(nuobjPath));
File.Copy(workingDirectory.PathTo(outobj), nuobjPath, true);
}
else
{
missingOutputs.Add(string.Format("Missing expected output {0}", outobj.getRelativePath()));
}
}
else
{
hasVirtualOutputs = true;
if (this.repository.GetDisposition(outobj) is Fresh)
{
// Nothing to cache; virtual objects only survive in the Repository, the in-process store.
}
else
{
missingOutputs.Add(string.Format("Missing expected virtual {0}", outobj.getRelativePath()));
}
}
}
if (!(executionDisposition is Failed) && missingOutputs.Count() > 0)
{
executionDisposition = new Failed(missingOutputs);
}
ResultSummaryRecord summary = new ResultSummaryRecord(verb, executionDisposition, outputs);
string inputHash = this.computeInputHash(verb, true);
Util.Assert(inputHash != null);
if (!hasVirtualOutputs)
{
this.repository.StoreResult(inputHash, summary);
}
else
{
this.Say("Not caching verb persistently: " + verb);
}
this.verbIsComplete(verb, executionDisposition);
return(executionDisposition);
}
