void markFailed(IVerb verb) { //- at least one of verb's inputs has a permanent failure, so we didn't //- even try to execute it. Disposition disposition = new Failed("upstream failure"); ResultSummaryRecord summary = new ResultSummaryRecord(verb, disposition, new BuildObjectValuePointer[] {}); //- NB never store upstream failures to the persistent cache, because //- they depend on our knowledge this run that the upstream verb failed. //- If, in another run, the verb or its inputs are modified, produce the //- same outputs, but returns success, we'll be stuck pulling this //- upstream failure out of cache but calling this verb a failure. //-string inputHash = computeInputHash(verb, false); //-if (inputHash != null) //-{ //- Util.Assert(false); //- //- //- //- //- //- resultCache.storeResult(inputHash, summary); //-} //-else //-{ unrecordableFailures[verb] = disposition; //-} verbIsComplete(verb, disposition, BlessingRequest.Bless); }
/// <summary> /// Fetches a result record from the cache. /// </summary> /// <param name="inputHash"> /// Item cache hash key the result is stored under. /// </param> /// <param name="includeFailedResults"> /// Whether to return cached failures as well as successes. /// </param> /// <returns> /// The result record requested if available, otherwise returns a /// summary record with Stale disposition. /// </returns> public ResultSummaryRecord FetchResult(string inputHash, bool includeFailedResults = false) { byte[] result = this.itemCache.FetchItem(ItemCacheContainer.Results, inputHash); if ((result == null) && includeFailedResults) { result = this.itemCache.FetchItem(ItemCacheContainer.FailedResults, inputHash); } if (result != null) { MemoryStream resultStream = new MemoryStream(result); try { using (StreamReader inReader = new StreamReader(resultStream)) { string xmlSummary = inReader.ReadToEnd(); return(ResultSummaryRecord.FromXml(xmlSummary)); } } catch (System.Xml.XmlException ex) { throw new ObjectMissingFromCacheException(inputHash, "Malformed xml: " + ex.ToString()); } finally { resultStream.Dispose(); } } else { return(new ResultSummaryRecord(new Stale(), new BuildObjectValuePointer[] { }, false)); } }
public void storeResult(string inputHash, ResultSummaryRecord result) { using (MemoryStream outStream = new MemoryStream()) { using (StreamWriter outWriter = new StreamWriter(outStream)) { outWriter.Write(result.toXml()); } this.itemCache.StoreItem(ItemCacheContainer.Results, inputHash, outStream.ToArray()); } }
/// <summary> /// Mark a verb as having failed. /// </summary> /// <param name="verb">The verb that failed.</param> private void markFailed(IVerb verb) { // At least one of verb's inputs has a permanent failure, so we didn't // even try to execute it. Disposition disposition = new Failed("upstream failure"); ResultSummaryRecord summary = new ResultSummaryRecord(verb, disposition, new BuildObjectValuePointer[] { }); // NB never store upstream failures to the persistent cache, because // they depend on our knowledge this run that the upstream verb failed. // If, in another run, the verb or its inputs are modified, produce the // same outputs, but returns success, we'll be stuck pulling this // upstream failure out of cache but calling this verb a failure. ////string inputHash = computeInputHash(verb, false); ////if (inputHash != null) ////{ //// Util.Assert(false); //// // "Upstream failures" will never have a computable inputHash, because their inputs //// // can't be considered known. Even if the upstream verb wrote something to disk, //// // what if the upstream verb changes to no longer fail but still emit the same thing? //// // We wouldn't want to conclude that, because the inputs hadn't changed, this //// // verb still had an upstream failure. //// repository.StoreResult(inputHash, summary); ////} ////else ////{ this.unrecordableFailures[verb] = disposition; ////} // Mark all the verb's outputs as Failed in the repository. foreach (BuildObject obj in verb.getOutputs()) { if (obj is VirtualBuildObject) { this.repository.StoreVirtual(obj, disposition, null); } else { this.repository.AddObject(obj, disposition, null); } } this.verbIsComplete(verb, disposition); }
/// <summary> /// Checks the cache for a previous execution of this verb operating on /// the same inputs; updates our state with the cached results if so. /// </summary> /// <param name="verb">The verb to check for prior execution.</param> /// <param name="inputHash"> /// An identifier for this verb operating on a specific set of inputs. /// </param> /// <returns> /// True if usable previous execution was found, false otherwise. /// </returns> private bool fetchFromCache(IVerb verb, string inputHash) { try { ResultSummaryRecord summary = this.repository.FetchResult(inputHash); if (summary.Disposition is Stale) { return(false); } // REVIEW: Since we aren't asking FetchResult to return failures, // this check is no longer needed. Or at least it won't be once // the "Results" cache is cleared of all existing failure records. if (this.rejectCachedFailures && (summary.Disposition is Failed || summary.IsVerificationTimeout)) { Logger.WriteLine(string.Format( "NOTE: rejecting failure from cache {0}", verb)); return(false); } this.Say(string.Format("disposeCurrentVerbs pulls {0} from cache", verb)); // Hey, this verb is already computed! Nothing to do. // Add the verb execution's results to the repository. this.repository.AddVerbResults(verb, summary); this.verbIsComplete(verb, summary.Disposition); return(true); } catch (ObjectMissingFromCacheException ex) { Logger.WriteLine(string.Format( "WARNING: expected object {0} missing from cache; discarding cached result {1}", ex, verb)); return(false); } }
/// <summary> /// Stores a result record in the cache. /// </summary> /// <param name="inputHash"> /// Item cache hash key to store result under. /// </param> /// <param name="result">The result record to store.</param> public void StoreResult(string inputHash, ResultSummaryRecord result) { ItemCacheContainer container; if (result.IsVerificationTimeout || (result.Disposition is Failed)) { container = ItemCacheContainer.FailedResults; } else { container = ItemCacheContainer.Results; } using (MemoryStream outStream = new MemoryStream()) { using (StreamWriter outWriter = new StreamWriter(outStream)) { outWriter.Write(result.ToXml()); } this.itemCache.StoreItem(container, inputHash, outStream.ToArray()); } }
private bool fetchFromCache(IVerb verb, string inputHash) { try { ResultSummaryRecord summary = resultCache.fetchResult(inputHash); if (summary.disposition is Stale) { return(false); } if (rejectCachedFailures && (summary.disposition is Failed || summary.isVerificationTimeout)) { Logger.WriteLine(String.Format( "NOTE: rejecting failure from cache {0}", verb)); return(false); } Say(String.Format("disposeCurrentVerbs pulls {0} from cache", verb)); //- Hey, this verb is already computed! Nothing to do. //- Bring the objects into nuobj where the user can see //- them -- and where we can explore them for dependency evaluation. resultCache.fetchOutputObjects(verb, summary.outputs, summary.disposition); verbIsComplete(verb, summary.disposition, BlessingRequest.NoBless); return(true); } catch (ObjectMissingFromCacheException ex) { Logger.WriteLine(String.Format( "WARNING: expected object {0} missing from cache; discarding cached result {1}", ex, verb)); return(false); } }
/// <summary> /// Adds the output objects from a cached verb execution to the /// repository, and ensures they are present in the item cache. /// </summary> /// <param name="verb">The verb whose outputs to add.</param> /// <param name="resultRecord"> /// The result summary record of the verb execution. /// </param> /// <remarks> /// Call only when output objects are known to be cached /// (i.e. because FetchResult returned non-Stale). /// REVIEW: This function probably shouldn't be in this file. /// It does something similar for cached verb results that /// the scheduler's recordResult method does for new verb /// executions. Move this there and/or refactor? /// </remarks> public void AddVerbResults(IVerb verb, ResultSummaryRecord resultRecord) { if (this.alreadyAddedVerbs.Contains(verb)) { // We only need to add a cached verb execution's outputs once. return; } Disposition disposition = resultRecord.Disposition; // REVIEW: In the below, some of these IEnumerables should be // HashSets, and the HashSet should be a simple List. // Create a collection of the potential outputs. IEnumerable <BuildObject> outputs = verb.getOutputs(); IEnumerable <BuildObject> failureOutputs = verb.getFailureOutputs(); outputs = outputs.Concat(failureOutputs); Dictionary <string, BuildObject> potentialOutputs = new Dictionary <string, BuildObject>(); foreach (BuildObject obj in outputs) { potentialOutputs.Add(obj.getRelativePath(), obj); } // Compare the actual outputs with the potential outputs, // and add the actual ones to the repository. HashSet <BuildObject> recorded = new HashSet <BuildObject>(); foreach (BuildObjectValuePointer actualOutput in resultRecord.Outputs) { if (potentialOutputs.ContainsKey(actualOutput.RelativePath)) { BuildObject obj = potentialOutputs[actualOutput.RelativePath]; // TODO: Verify that the object exists in the item cache! this.AddObject(obj, disposition, actualOutput.ObjectHash); recorded.Add(obj); // Store a copy of this verb output as a file in the real nuobj directory. Util.Assert(actualOutput.RelativePath.StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal)); this.itemCache.FetchItemToFile(ItemCacheContainer.Objects, actualOutput.ObjectHash, IronRootDirectory.PathTo(actualOutput.RelativePath)); } else { // Complain if we find interloping outputs. throw new Exception("Distressing: some actual verb outputs aren't in the verb's list of potential outputs"); } } // Create a collection of missing outputs. IEnumerable <BuildObject> unrecorded = outputs.Except(recorded).Except(failureOutputs); // For non-Failed verb runs, complain if all expected outputs don't // show up in the actual outputs. Util.Assert(unrecorded.Count() == 0 || disposition is Failed); // For cached verb runs with permanent failures (i.e. disposition // is Failed), we want to mark all of the expected outputs as Failed // even if no corresponding actual output was produced during the // failed verb run. foreach (BuildObject obj in unrecorded) { this.AddObject(obj, disposition, null); } // Remember that we've already added this verb's outputs. this.alreadyAddedVerbs.Add(verb); }
/// <summary> /// Adds the output objects from a cached verb execution to the /// repository, and ensures they are present in the item cache. /// </summary> /// <param name="verb">The verb whose outputs to add.</param> /// <param name="resultRecord"> /// The result summary record of the verb execution. /// </param> /// <remarks> /// Call only when output objects are known to be cached /// (i.e. because FetchResult returned non-Stale). /// REVIEW: This function probably shouldn't be in this file. /// It does something similar for cached verb results that /// the scheduler's recordResult method does for new verb /// executions. Move this there and/or refactor? /// </remarks> public void AddVerbResults(IVerb verb, ResultSummaryRecord resultRecord) { if (this.alreadyAddedVerbs.Contains(verb)) { // We only need to add a cached verb execution's outputs once. return; } Disposition disposition = resultRecord.Disposition; // REVIEW: In the below, some of these IEnumerables should be // HashSets, and the HashSet should be a simple List. // Create a collection of the potential outputs. IEnumerable<BuildObject> outputs = verb.getOutputs(); IEnumerable<BuildObject> failureOutputs = verb.getFailureOutputs(); outputs = outputs.Concat(failureOutputs); Dictionary<string, BuildObject> potentialOutputs = new Dictionary<string, BuildObject>(); foreach (BuildObject obj in outputs) { potentialOutputs.Add(obj.getRelativePath(), obj); } // Compare the actual outputs with the potential outputs, // and add the actual ones to the repository. HashSet<BuildObject> recorded = new HashSet<BuildObject>(); foreach (BuildObjectValuePointer actualOutput in resultRecord.Outputs) { if (potentialOutputs.ContainsKey(actualOutput.RelativePath)) { BuildObject obj = potentialOutputs[actualOutput.RelativePath]; // TODO: Verify that the object exists in the item cache! this.AddObject(obj, disposition, actualOutput.ObjectHash); recorded.Add(obj); // Store a copy of this verb output as a file in the real nuobj directory. Util.Assert(actualOutput.RelativePath.StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal)); this.itemCache.FetchItemToFile(ItemCacheContainer.Objects, actualOutput.ObjectHash, IronRootDirectory.PathTo(actualOutput.RelativePath)); } else { // Complain if we find interloping outputs. throw new Exception("Distressing: some actual verb outputs aren't in the verb's list of potential outputs"); } } // Create a collection of missing outputs. IEnumerable<BuildObject> unrecorded = outputs.Except(recorded).Except(failureOutputs); // For non-Failed verb runs, complain if all expected outputs don't // show up in the actual outputs. Util.Assert(unrecorded.Count() == 0 || disposition is Failed); // For cached verb runs with permanent failures (i.e. disposition // is Failed), we want to mark all of the expected outputs as Failed // even if no corresponding actual output was produced during the // failed verb run. foreach (BuildObject obj in unrecorded) { this.AddObject(obj, disposition, null); } // Remember that we've already added this verb's outputs. this.alreadyAddedVerbs.Add(verb); }
/// <summary> /// Stores a result record in the cache. /// </summary> /// <param name="inputHash"> /// Item cache hash key to store result under. /// </param> /// <param name="result">The result record to store.</param> public void StoreResult(string inputHash, ResultSummaryRecord result) { ItemCacheContainer container; if (result.IsVerificationTimeout || (result.Disposition is Failed)) { container = ItemCacheContainer.FailedResults; } else { container = ItemCacheContainer.Results; } using (MemoryStream outStream = new MemoryStream()) { using (StreamWriter outWriter = new StreamWriter(outStream)) { outWriter.Write(result.ToXml()); } this.itemCache.StoreItem(container, inputHash, outStream.ToArray()); } }
/// <summary> /// Mark a verb as having failed. /// </summary> /// <param name="verb">The verb that failed.</param> private void markFailed(IVerb verb) { // At least one of verb's inputs has a permanent failure, so we didn't // even try to execute it. Disposition disposition = new Failed("upstream failure"); ResultSummaryRecord summary = new ResultSummaryRecord(verb, disposition, new BuildObjectValuePointer[] { }); // NB never store upstream failures to the persistent cache, because // they depend on our knowledge this run that the upstream verb failed. // If, in another run, the verb or its inputs are modified, produce the // same outputs, but returns success, we'll be stuck pulling this // upstream failure out of cache but calling this verb a failure. ////string inputHash = computeInputHash(verb, false); ////if (inputHash != null) ////{ //// Util.Assert(false); //// // "Upstream failures" will never have a computable inputHash, because their inputs //// // can't be considered known. Even if the upstream verb wrote something to disk, //// // what if the upstream verb changes to no longer fail but still emit the same thing? //// // We wouldn't want to conclude that, because the inputs hadn't changed, this //// // verb still had an upstream failure. //// repository.StoreResult(inputHash, summary); ////} ////else ////{ this.unrecordableFailures[verb] = disposition; ////} // Mark all the verb's outputs as Failed in the repository. foreach (BuildObject obj in verb.getOutputs()) { if (obj is VirtualBuildObject) { this.repository.StoreVirtual(obj, disposition, null); } else { this.repository.AddObject(obj, disposition, null); } } this.verbIsComplete(verb, disposition); }
/// <summary> /// Record how each output of a task appeared. /// </summary> /// <param name="completion">The task completion notification.</param> /// <returns>Overall result of the verb execution.</returns> private Disposition recordResult(VerbRunner.TaskCompletion completion) { WorkingDirectory workingDirectory = completion.workingDirectory; IVerb verb = completion.verb; Disposition executionDisposition = completion.disposition; List<BuildObjectValuePointer> outputs = new List<BuildObjectValuePointer>(); List<string> missingOutputs = new List<string>(); IEnumerable<BuildObject> expectedOutputs = verb.getOutputs(); if (executionDisposition is Failed) { expectedOutputs = expectedOutputs.Concat(verb.getFailureOutputs()); } bool hasVirtualOutputs = false; foreach (BuildObject outobj in expectedOutputs) { if (!(outobj is VirtualBuildObject)) { // For expected file outputs, check for existence in working directory. // REVIEW: Add method to WorkingDirectory which does this? if (File.Exists(workingDirectory.PathTo(outobj))) { // Try to catch accidental case mismatches that would burn us when we // try to fetch the file back in. ////string fsname = PathNormalizer.dbg_normalizePath_nocache(outobj.deprecatedGetFilesystemPath(), false); ////Util.Assert(Path.GetFileName(fsname).Equals(outobj.getFileName())); // REVIEW: Do we need to worry about case mismatches anymore? See comments above. outputs.Add(this.repository.Store(workingDirectory, outobj, executionDisposition)); // Store a copy of this verb output as a file in the real nuobj directory. Util.Assert(outobj.getRelativePath().StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal)); string nuobjPath = IronRootDirectory.PathTo(outobj); Directory.CreateDirectory(Path.GetDirectoryName(nuobjPath)); File.Copy(workingDirectory.PathTo(outobj), nuobjPath, true); } else { missingOutputs.Add(string.Format("Missing expected output {0}", outobj.getRelativePath())); } } else { hasVirtualOutputs = true; if (this.repository.GetDisposition(outobj) is Fresh) { // Nothing to cache; virtual objects only survive in the Repository, the in-process store. } else { missingOutputs.Add(string.Format("Missing expected virtual {0}", outobj.getRelativePath())); } } } if (!(executionDisposition is Failed) && missingOutputs.Count() > 0) { executionDisposition = new Failed(missingOutputs); } ResultSummaryRecord summary = new ResultSummaryRecord(verb, executionDisposition, outputs); string inputHash = this.computeInputHash(verb, true); Util.Assert(inputHash != null); if (!hasVirtualOutputs) { this.repository.StoreResult(inputHash, summary); } else { this.Say("Not caching verb persistently: " + verb); } this.verbIsComplete(verb, executionDisposition); return executionDisposition; }
/// <summary> /// Record how each output of a task appeared. /// </summary> /// <param name="completion">The task completion notification.</param> /// <returns>Overall result of the verb execution.</returns> private Disposition recordResult(VerbRunner.TaskCompletion completion) { WorkingDirectory workingDirectory = completion.workingDirectory; IVerb verb = completion.verb; Disposition executionDisposition = completion.disposition; List <BuildObjectValuePointer> outputs = new List <BuildObjectValuePointer>(); List <string> missingOutputs = new List <string>(); IEnumerable <BuildObject> expectedOutputs = verb.getOutputs(); if (executionDisposition is Failed) { expectedOutputs = expectedOutputs.Concat(verb.getFailureOutputs()); } bool hasVirtualOutputs = false; foreach (BuildObject outobj in expectedOutputs) { if (!(outobj is VirtualBuildObject)) { // For expected file outputs, check for existence in working directory. // REVIEW: Add method to WorkingDirectory which does this? if (File.Exists(workingDirectory.PathTo(outobj))) { // Try to catch accidental case mismatches that would burn us when we // try to fetch the file back in. ////string fsname = PathNormalizer.dbg_normalizePath_nocache(outobj.deprecatedGetFilesystemPath(), false); ////Util.Assert(Path.GetFileName(fsname).Equals(outobj.getFileName())); // REVIEW: Do we need to worry about case mismatches anymore? See comments above. outputs.Add(this.repository.Store(workingDirectory, outobj, executionDisposition)); // Store a copy of this verb output as a file in the real nuobj directory. Util.Assert(outobj.getRelativePath().StartsWith(BuildEngine.theEngine.getObjRoot(), StringComparison.Ordinal)); string nuobjPath = IronRootDirectory.PathTo(outobj); Directory.CreateDirectory(Path.GetDirectoryName(nuobjPath)); File.Copy(workingDirectory.PathTo(outobj), nuobjPath, true); } else { missingOutputs.Add(string.Format("Missing expected output {0}", outobj.getRelativePath())); } } else { hasVirtualOutputs = true; if (this.repository.GetDisposition(outobj) is Fresh) { // Nothing to cache; virtual objects only survive in the Repository, the in-process store. } else { missingOutputs.Add(string.Format("Missing expected virtual {0}", outobj.getRelativePath())); } } } if (!(executionDisposition is Failed) && missingOutputs.Count() > 0) { executionDisposition = new Failed(missingOutputs); } ResultSummaryRecord summary = new ResultSummaryRecord(verb, executionDisposition, outputs); string inputHash = this.computeInputHash(verb, true); Util.Assert(inputHash != null); if (!hasVirtualOutputs) { this.repository.StoreResult(inputHash, summary); } else { this.Say("Not caching verb persistently: " + verb); } this.verbIsComplete(verb, executionDisposition); return(executionDisposition); }
Disposition recordResult(IVerb verb, Disposition executionDisposition) { //- record how each output appeared. List <BuildObjectValuePointer> outputs = new List <BuildObjectValuePointer>(); List <string> missingOutputs = new List <string>(); IEnumerable <BuildObject> expectedOutputs = verb.getOutputs(); if (executionDisposition is Failed) { expectedOutputs = expectedOutputs.Concat(verb.getFailureOutputs()); } bool hasVirtualOutputs = false; foreach (BuildObject outobj in expectedOutputs) { if (!(outobj is VirtualBuildObject)) { if (File.Exists(outobj.getFilesystemPath())) { //- Try to catch accidental case mismatches that would burn us when we //- try to fetch the file back in. string fsname = PathNormalizer.dbg_normalizePath_nocache(outobj.getFilesystemPath(), false); Util.Assert(Path.GetFileName(fsname).Equals(outobj.getFileName())); outputs.Add(resultCache.storeObject(outobj)); } else { missingOutputs.Add(String.Format("Missing expected output {0}", outobj.getRelativePath())); } } else { hasVirtualOutputs = true; if (nuObjContents.getDisposition(outobj) is Fresh) { //- nothing to cache; virtual objects only survive in nuObjContents, the in-process cache } else { missingOutputs.Add(String.Format("Missing expected virtual {0}", outobj.getRelativePath())); } } } if (!(executionDisposition is Failed) && missingOutputs.Count() > 0) { executionDisposition = new Failed(missingOutputs); } ResultSummaryRecord summary = new ResultSummaryRecord(verb, executionDisposition, outputs); string inputHash = computeInputHash(verb, true); Util.Assert(inputHash != null); if (!hasVirtualOutputs) { resultCache.storeResult(inputHash, summary); } else { Say("Not caching verb persistently: " + verb); } verbIsComplete(verb, executionDisposition, BlessingRequest.Bless); return(executionDisposition); }