/// <summary> /// Mark a verb as completed, and adjust our schedule accordingly. /// </summary> /// <param name="verb">The verb that completed.</param> /// <param name="disp">The disposition of the verb's execution.</param> private void verbIsComplete(IVerb verb, Disposition disp) { ////Say(string.Format(" {0} is complete: {1}", verb, dbgDisposition)); ////if (disp is Failed) ////{ //// // Failures can be hard to debug, since they don't leave any //// // output in nuobj/. So report these even if they aren't //// // built this run. //// emitRealtimeReport(verb, disp); ////} // Invariant: all of this verb's objs are non-Stale. foreach (BuildObject obj in verb.getOutputs()) { ////Say(string.Format(" waking {0}", obj)); IEnumerable <IVerb> wokenSet = this.waitIndex.awaken(obj); ////foreach (IVerb wokenVerb in wokenSet) ////{ //// //Say(string.Format(" {0} woken", wokenVerb)); ////} this.nextVerbs.UnionWith(wokenSet); } this.emitRealtimeReport(verb, disp); this.requiredVerbs.Remove(verb); this.completedVerbs.Add(verb); }
/// <summary> /// Adds a verb to the mapping of build objects to the verb which /// creates them. /// </summary> /// <remarks> /// TODO: Make this private. /// </remarks> /// <param name="verb">The verb to add.</param> internal void addVerb(IVerb verb) { if (!this.knownVerbs.Add(verb)) { // We've already added this verb. return; } // Add all verb outputs to the output-to-verb map. foreach (BuildObject obj in verb.getOutputs()) { if (this.outputToVerbMap.ContainsKey(obj)) { Util.Assert(this.outputToVerbMap[obj].Equals(verb)); } else { this.outputToVerbMap[obj] = verb; } } // Recursively add all the verbs this verb is dependent upon, // so that we have a complete index of outputs back to the // verbs that generate them. foreach (IVerb dependentVerb in verb.getVerbs()) { this.addVerb(dependentVerb); } }
void verbIsComplete(IVerb verb, Disposition disp, BlessingRequest blessingRequest) { emitRealtimeReport(verb, disp); //-Say(String.Format(" {0} is complete: {1}", verb, dbgDisposition)); //-if (disp is Failed) //-{ //- //- //- //- emitRealtimeReport(verb, disp); //-} //- invariant: all of this verb's objs are non-Stale. foreach (BuildObject obj in verb.getOutputs()) { if (blessingRequest == BlessingRequest.Bless) { nuObjContents.blessGeneral(obj, disp); } //-Say(String.Format(" waking {0}", obj)); IEnumerable <IVerb> wokenSet = waitIndex.awaken(obj); //-foreach (IVerb wokenVerb in wokenSet) //-{ //- //-} nextVerbs.UnionWith(wokenSet); } requiredVerbs.Remove(verb); completedVerbs.Add(verb); }
void prepareObjDirectory(IVerb verb) { foreach (BuildObject obj in verb.getOutputs()) { obj.prepareObjDirectory(); } }
/// <summary> /// Prepares the working directory tree for a verb's execution. /// </summary> /// <param name="verb">The verb whose execution we're preparing for.</param> private void PrepareForVerb(WorkingDirectory workingDirectory, IVerb verb) { // Debugging aide: write out the abstract id for this verb. File.WriteAllText(workingDirectory.PathTo("Debug.txt"), verb.getAbstractIdentifier().ToString()); Repository repository = BuildEngine.theEngine.Repository; // Copy all verb inputs from the item cache to here. DependencyDisposition ddisp; foreach (BuildObject input in verb.getDependencies(out ddisp)) { if (!(input is VirtualBuildObject)) { workingDirectory.CreateDirectoryFor(input); // REVIEW: No longer needed? repository.Fetch(workingDirectory, input); } } // Ensures that the directory tree for each of the verb's outputs exists. foreach (BuildObject output in verb.getOutputs()) { workingDirectory.CreateDirectoryFor(output); } }
public override IEnumerable <BuildObject> getDependencies(out DependencyDisposition ddisp) { ddisp = DependencyDisposition.Complete; //- I really don't care how many outputs the parent has; any one will //- link me to the paarent. IEnumerable <BuildObject> result = _parent.getOutputs(); Util.Assert(result.Count() > 0); return(result); }
void IHasher.addVerb(IVerb verb) { foreach (BuildObject obj in verb.getOutputs()) { if (outputToVerbMap.ContainsKey(obj)) { Util.Assert(outputToVerbMap[obj].Equals(verb)); } else { outputToVerbMap[obj] = verb; } } }
//- Contract: call only when output objects are known to be cached //- (because fetchResult returned non-Stale). public void fetchOutputObjects(IVerb verb, IEnumerable <BuildObjectValuePointer> values, Disposition disp) { if (this.alreadyFetchedVerbs.Contains(verb)) { return; } IEnumerable <BuildObject> objects = verb.getOutputs(); IEnumerable <BuildObject> failureObjs = verb.getFailureOutputs(); objects = objects.Concat(failureObjs); Dictionary <string, BuildObject> objectDict = new Dictionary <string, BuildObject>(); foreach (BuildObject obj in objects) { objectDict.Add(obj.getRelativePath(), obj); } HashSet <BuildObject> recorded = new HashSet <BuildObject>(); foreach (BuildObjectValuePointer value in values) { if (objectDict.ContainsKey(value.relativePath)) { BuildObject obj = objectDict[value.relativePath]; obj.prepareObjDirectory(); this.fetchObject(value, obj); nuObjectContents.bless(obj, value.objectHash, disp); recorded.Add(obj); } else { throw new Exception("Distressing: some BOVPs aren't in obj.getOutputs"); } } IEnumerable <BuildObject> unrecorded = objects.Except(recorded).Except(failureObjs); Util.Assert(unrecorded.Count() == 0 || disp is Failed); foreach (BuildObject obj in unrecorded) { nuObjectContents.bless(obj, null, disp); } this.alreadyFetchedVerbs.Add(verb); }
/// <summary> /// Mark a verb as having failed. /// </summary> /// <param name="verb">The verb that failed.</param> private void markFailed(IVerb verb) { // At least one of verb's inputs has a permanent failure, so we didn't // even try to execute it. Disposition disposition = new Failed("upstream failure"); ResultSummaryRecord summary = new ResultSummaryRecord(verb, disposition, new BuildObjectValuePointer[] { }); // NB never store upstream failures to the persistent cache, because // they depend on our knowledge this run that the upstream verb failed. // If, in another run, the verb or its inputs are modified, produce the // same outputs, but returns success, we'll be stuck pulling this // upstream failure out of cache but calling this verb a failure. ////string inputHash = computeInputHash(verb, false); ////if (inputHash != null) ////{ //// Util.Assert(false); //// // "Upstream failures" will never have a computable inputHash, because their inputs //// // can't be considered known. Even if the upstream verb wrote something to disk, //// // what if the upstream verb changes to no longer fail but still emit the same thing? //// // We wouldn't want to conclude that, because the inputs hadn't changed, this //// // verb still had an upstream failure. //// repository.StoreResult(inputHash, summary); ////} ////else ////{ this.unrecordableFailures[verb] = disposition; ////} // Mark all the verb's outputs as Failed in the repository. foreach (BuildObject obj in verb.getOutputs()) { if (obj is VirtualBuildObject) { this.repository.StoreVirtual(obj, disposition, null); } else { this.repository.AddObject(obj, disposition, null); } } this.verbIsComplete(verb, disposition); }
Disposition recordResult(IVerb verb, Disposition executionDisposition) { //- record how each output appeared. List <BuildObjectValuePointer> outputs = new List <BuildObjectValuePointer>(); List <string> missingOutputs = new List <string>(); IEnumerable <BuildObject> expectedOutputs = verb.getOutputs(); if (executionDisposition is Failed) { expectedOutputs = expectedOutputs.Concat(verb.getFailureOutputs()); } bool hasVirtualOutputs = false; foreach (BuildObject outobj in expectedOutputs) { if (!(outobj is VirtualBuildObject)) { if (File.Exists(outobj.getFilesystemPath())) { //- Try to catch accidental case mismatches that would burn us when we //- try to fetch the file back in. string fsname = PathNormalizer.dbg_normalizePath_nocache(outobj.getFilesystemPath(), false); Util.Assert(Path.GetFileName(fsname).Equals(outobj.getFileName())); outputs.Add(resultCache.storeObject(outobj)); } else { missingOutputs.Add(String.Format("Missing expected output {0}", outobj.getRelativePath())); } } else { hasVirtualOutputs = true; if (nuObjContents.getDisposition(outobj) is Fresh) { //- nothing to cache; virtual objects only survive in nuObjContents, the in-process cache } else { missingOutputs.Add(String.Format("Missing expected virtual {0}", outobj.getRelativePath())); } } } if (!(executionDisposition is Failed) && missingOutputs.Count() > 0) { executionDisposition = new Failed(missingOutputs); } ResultSummaryRecord summary = new ResultSummaryRecord(verb, executionDisposition, outputs); string inputHash = computeInputHash(verb, true); Util.Assert(inputHash != null); if (!hasVirtualOutputs) { resultCache.storeResult(inputHash, summary); } else { Say("Not caching verb persistently: " + verb); } verbIsComplete(verb, executionDisposition, BlessingRequest.Bless); return(executionDisposition); }
/// <summary> /// Adds the output objects from a cached verb execution to the /// repository, and ensures they are present in the item cache. /// </summary> /// <param name="verb">The verb whose outputs to add.</param> /// <param name="resultRecord"> /// The result summary record of the verb execution. /// </param> /// <remarks> /// Call only when output objects are known to be cached /// (i.e. because FetchResult returned non-Stale). /// REVIEW: This function probably shouldn't be in this file. /// It does something similar for cached verb results that /// the scheduler's recordResult method does for new verb /// executions. Move this there and/or refactor? /// </remarks> public void AddVerbResults(IVerb verb, ResultSummaryRecord resultRecord) { if (this.alreadyAddedVerbs.Contains(verb)) { // We only need to add a cached verb execution's outputs once. return; } Disposition disposition = resultRecord.Disposition; // REVIEW: In the below, some of these IEnumerables should be // HashSets, and the HashSet should be a simple List. // Create a collection of the potential outputs. IEnumerable <BuildObject> outputs = verb.getOutputs(); IEnumerable <BuildObject> failureOutputs = verb.getFailureOutputs(); outputs = outputs.Concat(failureOutputs); Dictionary <string, BuildObject> potentialOutputs = new Dictionary <string, BuildObject>(); foreach (BuildObject obj in outputs) { potentialOutputs.Add(obj.getRelativePath(), obj); } // Compare the actual outputs with the potential outputs, // and add the actual ones to the repository. HashSet <BuildObject> recorded = new HashSet <BuildObject>(); foreach (BuildObjectValuePointer actualOutput in resultRecord.Outputs) { if (potentialOutputs.ContainsKey(actualOutput.RelativePath)) { BuildObject obj = potentialOutputs[actualOutput.RelativePath]; // TODO: Verify that the object exists in the item cache! this.AddObject(obj, disposition, actualOutput.ObjectHash); recorded.Add(obj); // Store a copy of this verb output as a file in the real nuobj directory. Util.Assert(actualOutput.RelativePath.StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal)); this.itemCache.FetchItemToFile(ItemCacheContainer.Objects, actualOutput.ObjectHash, IronRootDirectory.PathTo(actualOutput.RelativePath)); } else { // Complain if we find interloping outputs. throw new Exception("Distressing: some actual verb outputs aren't in the verb's list of potential outputs"); } } // Create a collection of missing outputs. IEnumerable <BuildObject> unrecorded = outputs.Except(recorded).Except(failureOutputs); // For non-Failed verb runs, complain if all expected outputs don't // show up in the actual outputs. Util.Assert(unrecorded.Count() == 0 || disposition is Failed); // For cached verb runs with permanent failures (i.e. disposition // is Failed), we want to mark all of the expected outputs as Failed // even if no corresponding actual output was produced during the // failed verb run. foreach (BuildObject obj in unrecorded) { this.AddObject(obj, disposition, null); } // Remember that we've already added this verb's outputs. this.alreadyAddedVerbs.Add(verb); }
/// <summary> /// Adds the output objects from a cached verb execution to the /// repository, and ensures they are present in the item cache. /// </summary> /// <param name="verb">The verb whose outputs to add.</param> /// <param name="resultRecord"> /// The result summary record of the verb execution. /// </param> /// <remarks> /// Call only when output objects are known to be cached /// (i.e. because FetchResult returned non-Stale). /// REVIEW: This function probably shouldn't be in this file. /// It does something similar for cached verb results that /// the scheduler's recordResult method does for new verb /// executions. Move this there and/or refactor? /// </remarks> public void AddVerbResults(IVerb verb, ResultSummaryRecord resultRecord) { if (this.alreadyAddedVerbs.Contains(verb)) { // We only need to add a cached verb execution's outputs once. return; } Disposition disposition = resultRecord.Disposition; // REVIEW: In the below, some of these IEnumerables should be // HashSets, and the HashSet should be a simple List. // Create a collection of the potential outputs. IEnumerable<BuildObject> outputs = verb.getOutputs(); IEnumerable<BuildObject> failureOutputs = verb.getFailureOutputs(); outputs = outputs.Concat(failureOutputs); Dictionary<string, BuildObject> potentialOutputs = new Dictionary<string, BuildObject>(); foreach (BuildObject obj in outputs) { potentialOutputs.Add(obj.getRelativePath(), obj); } // Compare the actual outputs with the potential outputs, // and add the actual ones to the repository. HashSet<BuildObject> recorded = new HashSet<BuildObject>(); foreach (BuildObjectValuePointer actualOutput in resultRecord.Outputs) { if (potentialOutputs.ContainsKey(actualOutput.RelativePath)) { BuildObject obj = potentialOutputs[actualOutput.RelativePath]; // TODO: Verify that the object exists in the item cache! this.AddObject(obj, disposition, actualOutput.ObjectHash); recorded.Add(obj); // Store a copy of this verb output as a file in the real nuobj directory. Util.Assert(actualOutput.RelativePath.StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal)); this.itemCache.FetchItemToFile(ItemCacheContainer.Objects, actualOutput.ObjectHash, IronRootDirectory.PathTo(actualOutput.RelativePath)); } else { // Complain if we find interloping outputs. throw new Exception("Distressing: some actual verb outputs aren't in the verb's list of potential outputs"); } } // Create a collection of missing outputs. IEnumerable<BuildObject> unrecorded = outputs.Except(recorded).Except(failureOutputs); // For non-Failed verb runs, complain if all expected outputs don't // show up in the actual outputs. Util.Assert(unrecorded.Count() == 0 || disposition is Failed); // For cached verb runs with permanent failures (i.e. disposition // is Failed), we want to mark all of the expected outputs as Failed // even if no corresponding actual output was produced during the // failed verb run. foreach (BuildObject obj in unrecorded) { this.AddObject(obj, disposition, null); } // Remember that we've already added this verb's outputs. this.alreadyAddedVerbs.Add(verb); }
/// <summary> /// Mark a verb as completed, and adjust our schedule accordingly. /// </summary> /// <param name="verb">The verb that completed.</param> /// <param name="disp">The disposition of the verb's execution.</param> private void verbIsComplete(IVerb verb, Disposition disp) { ////Say(string.Format(" {0} is complete: {1}", verb, dbgDisposition)); ////if (disp is Failed) ////{ //// // Failures can be hard to debug, since they don't leave any //// // output in nuobj/. So report these even if they aren't //// // built this run. //// emitRealtimeReport(verb, disp); ////} // Invariant: all of this verb's objs are non-Stale. foreach (BuildObject obj in verb.getOutputs()) { ////Say(string.Format(" waking {0}", obj)); IEnumerable<IVerb> wokenSet = this.waitIndex.awaken(obj); ////foreach (IVerb wokenVerb in wokenSet) ////{ //// //Say(string.Format(" {0} woken", wokenVerb)); ////} this.nextVerbs.UnionWith(wokenSet); } this.emitRealtimeReport(verb, disp); this.requiredVerbs.Remove(verb); this.completedVerbs.Add(verb); }
/// <summary> /// Record how each output of a task appeared. /// </summary> /// <param name="completion">The task completion notification.</param> /// <returns>Overall result of the verb execution.</returns> private Disposition recordResult(VerbRunner.TaskCompletion completion) { WorkingDirectory workingDirectory = completion.workingDirectory; IVerb verb = completion.verb; Disposition executionDisposition = completion.disposition; List <BuildObjectValuePointer> outputs = new List <BuildObjectValuePointer>(); List <string> missingOutputs = new List <string>(); IEnumerable <BuildObject> expectedOutputs = verb.getOutputs(); if (executionDisposition is Failed) { expectedOutputs = expectedOutputs.Concat(verb.getFailureOutputs()); } bool hasVirtualOutputs = false; foreach (BuildObject outobj in expectedOutputs) { if (!(outobj is VirtualBuildObject)) { // For expected file outputs, check for existence in working directory. // REVIEW: Add method to WorkingDirectory which does this? if (File.Exists(workingDirectory.PathTo(outobj))) { // Try to catch accidental case mismatches that would burn us when we // try to fetch the file back in. ////string fsname = PathNormalizer.dbg_normalizePath_nocache(outobj.deprecatedGetFilesystemPath(), false); ////Util.Assert(Path.GetFileName(fsname).Equals(outobj.getFileName())); // REVIEW: Do we need to worry about case mismatches anymore? See comments above. outputs.Add(this.repository.Store(workingDirectory, outobj, executionDisposition)); // Store a copy of this verb output as a file in the real nuobj directory. Util.Assert(outobj.getRelativePath().StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal)); string nuobjPath = IronRootDirectory.PathTo(outobj); Directory.CreateDirectory(Path.GetDirectoryName(nuobjPath)); File.Copy(workingDirectory.PathTo(outobj), nuobjPath, true); } else { missingOutputs.Add(string.Format("Missing expected output {0}", outobj.getRelativePath())); } } else { hasVirtualOutputs = true; if (this.repository.GetDisposition(outobj) is Fresh) { // Nothing to cache; virtual objects only survive in the Repository, the in-process store. } else { missingOutputs.Add(string.Format("Missing expected virtual {0}", outobj.getRelativePath())); } } } if (!(executionDisposition is Failed) && missingOutputs.Count() > 0) { executionDisposition = new Failed(missingOutputs); } ResultSummaryRecord summary = new ResultSummaryRecord(verb, executionDisposition, outputs); string inputHash = this.computeInputHash(verb, true); Util.Assert(inputHash != null); if (!hasVirtualOutputs) { this.repository.StoreResult(inputHash, summary); } else { this.Say("Not caching verb persistently: " + verb); } this.verbIsComplete(verb, executionDisposition); return(executionDisposition); }