private RuntimeCacheMissAnalyzer(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
FingerprintStore previousFingerprintStore,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
CacheMissDiffFormat cacheMissDiffFormat,
bool cacheMissBatch,
FingerprintStoreTestHooks testHooks = null)
{
m_loggingContext = loggingContext;
m_logTarget = logTarget;
m_context = context;
PreviousFingerprintStore = previousFingerprintStore;
m_visitor = new NodeVisitor(graph);
m_changedPips = new VisitationTracker(graph);
m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>();
m_runnablePipPerformance = runnablePipPerformance;
m_cacheMissDiffFormat = cacheMissDiffFormat;
m_maxCacheMissCanPerform = cacheMissBatch ? EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value * EngineEnvironmentSettings.MaxMessagesPerBatch : EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value;
m_batchLoggingQueue = cacheMissBatch ? NagleQueue <JProperty> .Create(
BatchLogging,
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(1),
batchSize : EngineEnvironmentSettings.MaxMessagesPerBatch) : null;
m_testHooks = testHooks;
m_testHooks?.InitRuntimeCacheMisses();
}
/// <summary>
/// Analyzes the cache miss for a specific pip.
/// </summary>
public static CacheMissAnalysisDetailAndResult AnalyzeCacheMiss(
PipCacheMissInfo missInfo,
Func <PipRecordingSession> oldSessionFunc,
Func <PipRecordingSession> newSessionFunc,
CacheMissDiffFormat diffFormat)
{
Contract.Requires(oldSessionFunc != null);
Contract.Requires(newSessionFunc != null);
var cacheMissType = missInfo.CacheMissType.ToString();
var cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType);
switch (missInfo.CacheMissType)
{
// Fingerprint miss.
case PipCacheMissType.MissForDescriptorsDueToAugmentedWeakFingerprints:
case PipCacheMissType.MissForDescriptorsDueToWeakFingerprints:
case PipCacheMissType.MissForDescriptorsDueToStrongFingerprints:
// Compute the pip unique output hash to use as the primary lookup key for fingerprint store entries
cacheMissAnalysisDetailAndResult = AnalyzeFingerprints(oldSessionFunc, newSessionFunc, diffFormat, cacheMissType);
break;
// We had a weak and strong fingerprint match, but couldn't retrieve correct data from the cache
case PipCacheMissType.MissForCacheEntry:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.DataMiss, "Cache entry missing from the cache.");
break;
case PipCacheMissType.MissForProcessMetadata:
case PipCacheMissType.MissForProcessMetadataFromHistoricMetadata:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.DataMiss, "MetaData missing from the cache.");
break;
case PipCacheMissType.MissForProcessOutputContent:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.OutputMiss, "Outputs missing from the cache.", new JObject(new JProperty("MissingOutputs", missInfo.MissedOutputs)));
break;
case PipCacheMissType.MissDueToInvalidDescriptors:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.InvalidDescriptors, "Cache returned invalid data.");
break;
case PipCacheMissType.MissForDescriptorsDueToArtificialMissOptions:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.ArtificialMiss, "Cache miss artificially forced by user.");
break;
case PipCacheMissType.Hit:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.NoMiss, "Pip was a cache hit.");
break;
case PipCacheMissType.Invalid:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.Invalid, "No valid changes or cache issues were detected to cause process execution, but a process still executed.");
break;
default:
break;
}
return(cacheMissAnalysisDetailAndResult);
}
/// <summary>
/// Analyzes the cache miss for a specific pip.
/// </summary>
public static CacheMissAnalysisResult AnalyzeCacheMiss(
TextWriter writer,
PipCacheMissInfo missInfo,
Func <PipRecordingSession> oldSessionFunc,
Func <PipRecordingSession> newSessionFunc,
CacheMissDiffFormat diffFormat)
{
Contract.Requires(oldSessionFunc != null);
Contract.Requires(newSessionFunc != null);
WriteLine($"Cache miss type: {missInfo.CacheMissType}", writer);
WriteLine(string.Empty, writer);
switch (missInfo.CacheMissType)
{
// Fingerprint miss
case PipCacheMissType.MissForDescriptorsDueToWeakFingerprints:
case PipCacheMissType.MissForDescriptorsDueToStrongFingerprints:
// Compute the pip unique output hash to use as the primary lookup key for fingerprint store entries
return(AnalyzeFingerprints(oldSessionFunc, newSessionFunc, writer, diffFormat));
// We had a weak and strong fingerprint match, but couldn't retrieve correct data from the cache
case PipCacheMissType.MissForCacheEntry:
WriteLine($"Cache entry missing from the cache.", writer);
return(CacheMissAnalysisResult.DataMiss);
case PipCacheMissType.MissForProcessMetadata:
case PipCacheMissType.MissForProcessMetadataFromHistoricMetadata:
WriteLine($"MetaData missing from the cache.", writer);
return(CacheMissAnalysisResult.DataMiss);
case PipCacheMissType.MissForProcessOutputContent:
WriteLine(new JProperty("MissingOutputs", missInfo.MissedOutputs).ToString(), writer);
return(CacheMissAnalysisResult.OutputMiss);
case PipCacheMissType.MissDueToInvalidDescriptors:
WriteLine($"Cache returned invalid data.", writer);
return(CacheMissAnalysisResult.InvalidDescriptors);
case PipCacheMissType.MissForDescriptorsDueToArtificialMissOptions:
WriteLine($"Cache miss artificially forced by user.", writer);
return(CacheMissAnalysisResult.ArtificialMiss);
case PipCacheMissType.Invalid:
WriteLine($"Unexpected condition! No valid changes or cache issues were detected to cause process execution, but a process still executed.", writer);
return(CacheMissAnalysisResult.Invalid);
case PipCacheMissType.Hit:
WriteLine($"Pip was a cache hit.", writer);
return(CacheMissAnalysisResult.NoMiss);
default:
WriteLine($"Unexpected condition! Unknown cache miss type.", writer);
return(CacheMissAnalysisResult.Invalid);
}
}
public Analyzer InitializeFingerprintStoreAnalyzer(AnalysisInput oldAnalysisInput, AnalysisInput newAnalysisInput)
{
string outputDirectory = null;
bool allPips = false;
bool noBanner = false;
long sshValue = -1;
CacheMissDiffFormat cacheMissDiffFormat = CacheMissDiffFormat.CustomJsonDiff;
foreach (var opt in AnalyzerOptions)
{
if (opt.Name.Equals("outputDirectory", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("o", StringComparison.OrdinalIgnoreCase))
{
outputDirectory = ParseSingletonPathOption(opt, outputDirectory);
}
else if (opt.Name.Equals("pip", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("p", StringComparison.OrdinalIgnoreCase))
{
sshValue = ParseSemistableHash(opt);
}
else if (opt.Name.StartsWith("allPips", StringComparison.OrdinalIgnoreCase))
{
allPips = ParseBooleanOption(opt);
}
else if (opt.Name.Equals("nobanner", StringComparison.OrdinalIgnoreCase))
{
noBanner = ParseBooleanOption(opt);
}
else if (opt.Name.Equals("cacheMissDiffFormat", StringComparison.OrdinalIgnoreCase))
{
cacheMissDiffFormat = ParseEnumOption <CacheMissDiffFormat>(opt);
}
else
{
throw Error("Unknown option for cache miss analysis: {0}", opt.Name);
}
}
if (string.IsNullOrEmpty(outputDirectory))
{
throw new Exception("'outputDirectory' is required.");
}
if (allPips && sshValue != -1)
{
throw new Exception("'allPips' can't be true if pipId is set.");
}
return(new FingerprintStoreAnalyzer(oldAnalysisInput, newAnalysisInput)
{
OutputDirectory = outputDirectory,
AllPips = allPips,
SemiStableHashToRun = sshValue,
NoBanner = noBanner
});
}
private RuntimeCacheMissAnalyzer(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
FingerprintStore previousFingerprintStore,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
CacheMissDiffFormat cacheMissDiffFormat)
{
m_loggingContext = loggingContext;
m_logTarget = logTarget;
m_context = context;
PreviousFingerprintStore = previousFingerprintStore;
m_visitor = new NodeVisitor(graph);
m_changedPips = new VisitationTracker(graph);
m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>();
m_runnablePipPerformance = runnablePipPerformance;
m_cacheMissDiffFormat = cacheMissDiffFormat;
}
private static CacheMissAnalysisResult AnalyzeFingerprints(
Func <PipRecordingSession> oldSessionFunc,
Func <PipRecordingSession> newSessionFunc,
TextWriter writer,
CacheMissDiffFormat diffFormat)
{
var result = CacheMissAnalysisResult.Invalid;
// While a PipRecordingSession is in scope, any pip information retrieved from the fingerprint store is
// automatically written out to per-pip files.
using (var oldPipSession = oldSessionFunc())
using (var newPipSession = newSessionFunc())
{
bool missingPipEntry = false;
if (!oldPipSession.EntryExists)
{
WriteLine("No fingerprint computation data found from old build.", writer, oldPipSession.PipWriter);
WriteLine("This may be the first execution where pip outputs were stored to the cache.", writer, oldPipSession.PipWriter);
// Write to just the old pip file
WriteLine(RepeatedStrings.DisallowedFileAccessesOrPipFailuresPreventCaching, oldPipSession.PipWriter);
missingPipEntry = true;
result = CacheMissAnalysisResult.MissingFromOldBuild;
WriteLine(string.Empty, writer, oldPipSession.PipWriter);
}
if (!newPipSession.EntryExists)
{
// Cases:
// 1. ScheduleProcessNotStoredToCacheDueToFileMonitoringViolations
// 2. ScheduleProcessNotStoredDueToMissingOutputs
// 3. ScheduleProcessNotStoredToWarningsUnderWarnAsError
// 4. ScheduleProcessNotStoredToCacheDueToInherentUncacheability
WriteLine("No fingerprint computation data found from new build.", writer, newPipSession.PipWriter);
// Write to just the new pip file
WriteLine(RepeatedStrings.DisallowedFileAccessesOrPipFailuresPreventCaching, newPipSession.PipWriter);
missingPipEntry = true;
result = CacheMissAnalysisResult.MissingFromNewBuild;
WriteLine(string.Empty, writer, newPipSession.PipWriter);
}
if (missingPipEntry)
{
// Only write once to the analysis file
WriteLine(RepeatedStrings.DisallowedFileAccessesOrPipFailuresPreventCaching, writer);
// Nothing to compare if an entry is missing
return(result);
}
if (oldPipSession.FormattedSemiStableHash != newPipSession.FormattedSemiStableHash)
{
// Make trivial json so the print looks like the rest of the diff
if (diffFormat == CacheMissDiffFormat.CustomJsonDiff)
{
var diff = new JProperty("SemiStableHash",
new JObject(
new JProperty("Old", oldPipSession.FormattedSemiStableHash),
new JProperty("New", newPipSession.FormattedSemiStableHash)));
WriteLine(new JObject(diff).ToString(), writer);
}
else
{
var oldNode = new JsonNode
{
Name = RepeatedStrings.FormattedSemiStableHashChanged
};
oldNode.Values.Add(oldPipSession.FormattedSemiStableHash);
var newNode = new JsonNode
{
Name = RepeatedStrings.FormattedSemiStableHashChanged
};
newNode.Values.Add(newPipSession.FormattedSemiStableHash);
WriteLine(JsonTree.PrintTreeDiff(oldNode, newNode), writer);
}
}
// Diff based off the actual fingerprints instead of the PipCacheMissType
// to avoid shared cache diff confusion.
//
// In the following shared cache scenario:
// Local cache: WeakFingerprint miss
// Remote cache: WeakFingerprint hit, StrongFingerprint miss
//
// The pip cache miss type will be a strong fingerprint miss,
// but the data in the fingerprint store will not match the
// remote cache's, so we diff based off what we have in the fingerprint store.
if (oldPipSession.WeakFingerprint != newPipSession.WeakFingerprint)
{
WriteLine("WeakFingerprint", writer);
if (diffFormat == CacheMissDiffFormat.CustomJsonDiff)
{
WriteLine(oldPipSession.DiffWeakFingerprint(newPipSession).ToString(), writer);
}
else
{
WriteLine(JsonTree.PrintTreeDiff(oldPipSession.GetWeakFingerprintTree(), newPipSession.GetWeakFingerprintTree()), writer);
}
result = CacheMissAnalysisResult.WeakFingerprintMismatch;
}
else if (oldPipSession.PathSetHash != newPipSession.PathSetHash)
{
WriteLine($"PathSet", writer);
if (diffFormat == CacheMissDiffFormat.CustomJsonDiff)
{
WriteLine(oldPipSession.DiffPathSet(newPipSession).ToString(), writer);
}
else
{
// JsonPatchDiff does not have pathset comparison.
WriteLine(JsonTree.PrintTreeDiff(oldPipSession.GetStrongFingerprintTree(), newPipSession.GetStrongFingerprintTree()), writer);
}
result = CacheMissAnalysisResult.PathSetHashMismatch;
}
else if (oldPipSession.StrongFingerprint != newPipSession.StrongFingerprint)
{
WriteLine("StrongFingerprint", writer);
if (diffFormat == CacheMissDiffFormat.CustomJsonDiff)
{
WriteLine(oldPipSession.DiffStrongFingerprint(newPipSession).ToString(), writer);
}
else
{
WriteLine(JsonTree.PrintTreeDiff(oldPipSession.GetStrongFingerprintTree(), newPipSession.GetStrongFingerprintTree()), writer);
}
result = CacheMissAnalysisResult.StrongFingerprintMismatch;
}
else
{
WriteLine("The fingerprints from both builds matched and no cache retrieval errors occurred.", writer);
WriteLine(RepeatedStrings.DisallowedFileAccessesOrPipFailuresPreventCaching, writer, oldPipSession.PipWriter, newPipSession.PipWriter);
result = CacheMissAnalysisResult.UncacheablePip;
}
}
return(result);
}
private static CacheMissAnalysisDetailAndResult AnalyzeFingerprints(
Func <PipRecordingSession> oldSessionFunc,
Func <PipRecordingSession> newSessionFunc,
CacheMissDiffFormat diffFormat,
string cacheMissType)
{
var cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType);
// While a PipRecordingSession is in scope, any pip information retrieved from the fingerprint store is
// automatically written out to per-pip files.
using (var oldPipSession = oldSessionFunc())
using (var newPipSession = newSessionFunc())
{
if (!oldPipSession.EntryExists)
{
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.MissingFromOldBuild, $"No fingerprint computation data found from old build. This may be the first execution where pip outputs were stored to the cache. {RepeatedStrings.DisallowedFileAccessesOrPipFailuresPreventCaching}");
// Write to just the old pip file
WriteLine(RepeatedStrings.DisallowedFileAccessesOrPipFailuresPreventCaching, oldPipSession.PipWriter);
WriteLine(string.Empty, oldPipSession.PipWriter);
return(cacheMissAnalysisDetailAndResult);
}
if (!newPipSession.EntryExists)
{
// Cases:
// 1. ScheduleProcessNotStoredToCacheDueToFileMonitoringViolations
// 2. ScheduleProcessNotStoredDueToMissingOutputs
// 3. ScheduleProcessNotStoredToWarningsUnderWarnAsError
// 4. ScheduleProcessNotStoredToCacheDueToInherentUncacheability
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.MissingFromNewBuild, $"No fingerprint computation data found from new build. {RepeatedStrings.DisallowedFileAccessesOrPipFailuresPreventCaching}");
// Write to just the new pip file
WriteLine(RepeatedStrings.DisallowedFileAccessesOrPipFailuresPreventCaching, newPipSession.PipWriter);
WriteLine(string.Empty, newPipSession.PipWriter);
return(cacheMissAnalysisDetailAndResult);
}
if (oldPipSession.FormattedSemiStableHash != newPipSession.FormattedSemiStableHash)
{
cacheMissAnalysisDetailAndResult.Detail.ReasonFromAnalysis = "SemiStableHashs of the builds are different.";
// Make trivial json so the print looks like the rest of the diff
if (diffFormat == CacheMissDiffFormat.CustomJsonDiff)
{
cacheMissAnalysisDetailAndResult.Detail.AddPropertyToCacheMissInfo("SemiStableHash",
new JObject(
new JProperty("Old", oldPipSession.FormattedSemiStableHash),
new JProperty("New", newPipSession.FormattedSemiStableHash)));
}
else
{
var oldNode = new JsonNode
{
Name = RepeatedStrings.FormattedSemiStableHashChanged
};
oldNode.Values.Add(oldPipSession.FormattedSemiStableHash);
var newNode = new JsonNode
{
Name = RepeatedStrings.FormattedSemiStableHashChanged
};
newNode.Values.Add(newPipSession.FormattedSemiStableHash);
cacheMissAnalysisDetailAndResult.Detail.AddPropertyToCacheMissInfo("SemiStableHash", JsonTree.PrintTreeDiff(oldNode, newNode));
}
}
// Diff based off the actual fingerprints instead of the PipCacheMissType
// The actual miss type can mismatch the reason from analysis, since cache miss analyzer is a best effort approach.
// There are several circumstances that fingerprintstore doesn't not match the real cache for pip's cache lookup.
// 1. Fingerprint store used for cache miss analysis is load at the beginning of the build and store at the end of the build,
// while cache entries in cache is store and retrieve during the build. There is a chance that the entry of a pip retrieved
// in cache look up time is not the same entry in the fingerprint store for analysis.
// Example: PipA executes in three builds in sequence Build1, Build2, Build3. Build1 finished before Build2 and Build3 starts.
// So, the fingerprint store created/updated in Build1 get stored in cache.
// Then Build2 and Build 3 starts and load the fingerprint store for cache miss analysis.
// PipA in Build2 finished execution and get stored in cache.
// PipB in Build3 started to execute. It get cache miss.
// Its actual cache miss type is from the cache look up result. In this example is the result compare to PipA in Build2.
// However, the cache miss analysis result is a comparison between PipA in Build1 and PipA in Build3.
// So, the ActualMissType can mismatch ReasonFromAnalysis.
// 2. Not all builds contribute to cache enable cache miss analysis, so the fingerprint store doesn't not have a full collection of entry from all builds.
if (oldPipSession.WeakFingerprint != newPipSession.WeakFingerprint)
{
cacheMissAnalysisDetailAndResult.Detail.ReasonFromAnalysis = "WeakFingerprints of the builds are different.";
if (diffFormat == CacheMissDiffFormat.CustomJsonDiff)
{
cacheMissAnalysisDetailAndResult.Detail.AddPropertyToCacheMissInfo("WeakFingerprintMismatchResult", oldPipSession.DiffWeakFingerprint(newPipSession));
}
else
{
cacheMissAnalysisDetailAndResult.Detail.AddPropertyToCacheMissInfo("WeakFingerprintMismatchResult", PrintTreeDiff(oldPipSession.GetWeakFingerprintTree(), newPipSession.GetWeakFingerprintTree(), oldPipSession));
}
cacheMissAnalysisDetailAndResult.Result = CacheMissAnalysisResult.WeakFingerprintMismatch;
}
else if (oldPipSession.PathSetHash != newPipSession.PathSetHash)
{
cacheMissAnalysisDetailAndResult.Detail.ReasonFromAnalysis = "PathSets of the builds are different.";
if (diffFormat == CacheMissDiffFormat.CustomJsonDiff)
{
cacheMissAnalysisDetailAndResult.Detail.AddPropertyToCacheMissInfo("PathSetMismatchResult", oldPipSession.DiffPathSet(newPipSession));
}
else
{
cacheMissAnalysisDetailAndResult.Detail.AddPropertyToCacheMissInfo("PathSetMismatchResult", PrintTreeDiff(oldPipSession.GetStrongFingerprintTree(), newPipSession.GetStrongFingerprintTree(), oldPipSession));
}
cacheMissAnalysisDetailAndResult.Result = CacheMissAnalysisResult.PathSetHashMismatch;
}
else if (oldPipSession.StrongFingerprint != newPipSession.StrongFingerprint)
{
cacheMissAnalysisDetailAndResult.Detail.ReasonFromAnalysis = "StrongFingerprints of the builds are different.";
if (diffFormat == CacheMissDiffFormat.CustomJsonDiff)
{
cacheMissAnalysisDetailAndResult.Detail.AddPropertyToCacheMissInfo("StrongFingerprintMismatchResult", oldPipSession.DiffStrongFingerprint(newPipSession));
}
else
{
cacheMissAnalysisDetailAndResult.Detail.AddPropertyToCacheMissInfo("StrongFingerprintMismatchResult", PrintTreeDiff(oldPipSession.GetStrongFingerprintTree(), newPipSession.GetStrongFingerprintTree(), oldPipSession));
}
cacheMissAnalysisDetailAndResult.Result = CacheMissAnalysisResult.StrongFingerprintMismatch;
}
else
{
cacheMissAnalysisDetailAndResult.Detail.ReasonFromAnalysis = $"The fingerprints from both builds matched and no cache retrieval errors occurred. {RepeatedStrings.DisallowedFileAccessesOrPipFailuresPreventCaching}";
WriteLine(RepeatedStrings.DisallowedFileAccessesOrPipFailuresPreventCaching, oldPipSession.PipWriter, newPipSession.PipWriter);
cacheMissAnalysisDetailAndResult.Result = CacheMissAnalysisResult.UncacheablePip;
}
}
return(cacheMissAnalysisDetailAndResult);
}
