/// <summary>
/// Analyzes the cache miss for a specific pip.
/// </summary>
public static CacheMissAnalysisDetailAndResult AnalyzeCacheMiss(
PipCacheMissInfo missInfo,
Func <PipRecordingSession> oldSessionFunc,
Func <PipRecordingSession> newSessionFunc,
CacheMissDiffFormat diffFormat)
{
Contract.Requires(oldSessionFunc != null);
Contract.Requires(newSessionFunc != null);
var cacheMissType = missInfo.CacheMissType.ToString();
var cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType);
switch (missInfo.CacheMissType)
{
// Fingerprint miss.
case PipCacheMissType.MissForDescriptorsDueToAugmentedWeakFingerprints:
case PipCacheMissType.MissForDescriptorsDueToWeakFingerprints:
case PipCacheMissType.MissForDescriptorsDueToStrongFingerprints:
// Compute the pip unique output hash to use as the primary lookup key for fingerprint store entries
cacheMissAnalysisDetailAndResult = AnalyzeFingerprints(oldSessionFunc, newSessionFunc, diffFormat, cacheMissType);
break;
// We had a weak and strong fingerprint match, but couldn't retrieve correct data from the cache
case PipCacheMissType.MissForCacheEntry:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.DataMiss, "Cache entry missing from the cache.");
break;
case PipCacheMissType.MissForProcessMetadata:
case PipCacheMissType.MissForProcessMetadataFromHistoricMetadata:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.DataMiss, "MetaData missing from the cache.");
break;
case PipCacheMissType.MissForProcessOutputContent:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.OutputMiss, "Outputs missing from the cache.", new JObject(new JProperty("MissingOutputs", missInfo.MissedOutputs)));
break;
case PipCacheMissType.MissDueToInvalidDescriptors:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.InvalidDescriptors, "Cache returned invalid data.");
break;
case PipCacheMissType.MissForDescriptorsDueToArtificialMissOptions:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.ArtificialMiss, "Cache miss artificially forced by user.");
break;
case PipCacheMissType.Hit:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.NoMiss, "Pip was a cache hit.");
break;
case PipCacheMissType.Invalid:
cacheMissAnalysisDetailAndResult = new CacheMissAnalysisDetailAndResult(cacheMissType, CacheMissAnalysisResult.Invalid, "No valid changes or cache issues were detected to cause process execution, but a process still executed.");
break;
default:
break;
}
return(cacheMissAnalysisDetailAndResult);
}
/// <summary>
/// Analyzes the cache miss for a specific pip.
/// </summary>
public static CacheMissAnalysisResult AnalyzeCacheMiss(
TextWriter writer,
PipCacheMissInfo missInfo,
Func <PipRecordingSession> oldSessionFunc,
Func <PipRecordingSession> newSessionFunc,
CacheMissDiffFormat diffFormat)
{
Contract.Requires(oldSessionFunc != null);
Contract.Requires(newSessionFunc != null);
WriteLine($"Cache miss type: {missInfo.CacheMissType}", writer);
WriteLine(string.Empty, writer);
switch (missInfo.CacheMissType)
{
// Fingerprint miss
case PipCacheMissType.MissForDescriptorsDueToWeakFingerprints:
case PipCacheMissType.MissForDescriptorsDueToStrongFingerprints:
// Compute the pip unique output hash to use as the primary lookup key for fingerprint store entries
return(AnalyzeFingerprints(oldSessionFunc, newSessionFunc, writer, diffFormat));
// We had a weak and strong fingerprint match, but couldn't retrieve correct data from the cache
case PipCacheMissType.MissForCacheEntry:
WriteLine($"Cache entry missing from the cache.", writer);
return(CacheMissAnalysisResult.DataMiss);
case PipCacheMissType.MissForProcessMetadata:
case PipCacheMissType.MissForProcessMetadataFromHistoricMetadata:
WriteLine($"MetaData missing from the cache.", writer);
return(CacheMissAnalysisResult.DataMiss);
case PipCacheMissType.MissForProcessOutputContent:
WriteLine(new JProperty("MissingOutputs", missInfo.MissedOutputs).ToString(), writer);
return(CacheMissAnalysisResult.OutputMiss);
case PipCacheMissType.MissDueToInvalidDescriptors:
WriteLine($"Cache returned invalid data.", writer);
return(CacheMissAnalysisResult.InvalidDescriptors);
case PipCacheMissType.MissForDescriptorsDueToArtificialMissOptions:
WriteLine($"Cache miss artificially forced by user.", writer);
return(CacheMissAnalysisResult.ArtificialMiss);
case PipCacheMissType.Invalid:
WriteLine($"Unexpected condition! No valid changes or cache issues were detected to cause process execution, but a process still executed.", writer);
return(CacheMissAnalysisResult.Invalid);
case PipCacheMissType.Hit:
WriteLine($"Pip was a cache hit.", writer);
return(CacheMissAnalysisResult.NoMiss);
default:
WriteLine($"Unexpected condition! Unknown cache miss type.", writer);
return(CacheMissAnalysisResult.Invalid);
}
}
private void AnalyzePip(PipCacheMissInfo miss, Process pip, TextWriter writer)
{
string pipUniqueOutputHashStr = null;
if ((pip as Process).TryComputePipUniqueOutputHash(PathTable, out var pipUniqueOutputHash, m_model.CachedGraph.MountPathExpander))
{
pipUniqueOutputHashStr = pipUniqueOutputHash.ToString();
}
WriteLine(pip.GetDescription(PipGraph.Context));
var analysisResult = CacheMissAnalysisResult.Invalid;
if (m_newCacheLookupReader != null &&
miss.CacheMissType == PipCacheMissType.MissForDescriptorsDueToStrongFingerprints &&
m_newCacheLookupReader.Store.ContainsFingerprintStoreEntry(pip.FormattedSemiStableHash, pipUniqueOutputHashStr))
{
// Strong fingerprint miss analysis is most accurate when compared to the fingerprints computed at cache lookup time
// because those fingerprints capture the state of the disk at cache lookup time, including dynamic observations
var resultAndDetail = CacheMissAnalysisUtilities.AnalyzeCacheMiss(
miss,
() => m_oldReader.StartPipRecordingSession(pip, pipUniqueOutputHashStr),
() => m_newCacheLookupReader.StartPipRecordingSession(pip, pipUniqueOutputHashStr),
CacheMissDiffFormat);
analysisResult = resultAndDetail.Result;
m_writer?.WriteLine(JsonConvert.SerializeObject(resultAndDetail.Detail));
}
else
{
var resultAndDetail = CacheMissAnalysisUtilities.AnalyzeCacheMiss(
miss,
() => m_oldReader.StartPipRecordingSession(pip, pipUniqueOutputHashStr),
() => m_newReader.StartPipRecordingSession(pip, pipUniqueOutputHashStr),
CacheMissDiffFormat);
analysisResult = resultAndDetail.Result;
m_writer?.WriteLine(JsonConvert.SerializeObject(resultAndDetail.Detail));
}
if (analysisResult == CacheMissAnalysisResult.MissingFromOldBuild)
{
Tracing.Logger.Log.FingerprintStorePipMissingFromOldBuild(LoggingContext);
}
else if (analysisResult == CacheMissAnalysisResult.MissingFromNewBuild)
{
Tracing.Logger.Log.FingerprintStorePipMissingFromNewBuild(LoggingContext);
}
else if (analysisResult == CacheMissAnalysisResult.UncacheablePip)
{
Tracing.Logger.Log.FingerprintStoreUncacheablePipAnalyzed(LoggingContext);
}
}
private void AnalyzePip(PipCacheMissInfo miss, Process pip, TextWriter writer)
{
(pip as Process).TryComputePipUniqueOutputHash(PathTable, out var pipUniqueOutputHash, m_model.CachedGraph.MountPathExpander);
WriteLine(pip.GetDescription(PipGraph.Context));
var result = CacheMissAnalysisUtilities.AnalyzeCacheMiss(
writer,
miss,
() => m_oldReader.StartPipRecordingSession(pip, pipUniqueOutputHash.ToString()),
() => m_newReader.StartPipRecordingSession(pip, pipUniqueOutputHash.ToString()));
if (result == CacheMissAnalysisResult.MissingFromOldBuild)
{
Tracing.Logger.Log.FingerprintStorePipMissingFromOldBuild(LoggingContext);
}
else if (result == CacheMissAnalysisResult.MissingFromNewBuild)
{
Tracing.Logger.Log.FingerprintStorePipMissingFromNewBuild(LoggingContext);
}
else if (result == CacheMissAnalysisResult.UncacheablePip)
{
Tracing.Logger.Log.FingerprintStoreUncacheablePipAnalyzed(LoggingContext);
}
}
internal void AddCacheMiss(PipCacheMissInfo cacheMissInfo)
{
m_pipCacheMissesDict.Add(cacheMissInfo.PipId, cacheMissInfo);
}
/// <summary>
/// Analyzes cache misses.
/// </summary>
public override int Analyze()
{
Console.WriteLine("Starting fingerprint store analysis at: " + DateTime.Now.ToString());
if (!m_newReader.TryGetCacheMissList(out var cacheMissList))
{
WriteLine("Could not retrieve cache miss list for analysis. There may have been a failure during the build.");
return(0);
}
Console.WriteLine("Finished getting list of pips that had a cache miss at: " + DateTime.Now.ToString());
if (m_oldReader.StoreVersion != m_newReader.StoreVersion)
{
WriteLine($"WARNING: Format version numbers of the fingerprint store do not match. Old: {m_oldReader.StoreVersion}, New: {m_newReader.StoreVersion}.");
}
if (SemiStableHashToRun != -1)
{
var firstMiss = cacheMissList.FirstOrDefault(x => PipTable.GetPipSemiStableHash(x.PipId) == SemiStableHashToRun);
if (firstMiss.CacheMissType == PipCacheMissType.Invalid)
{
foreach (var pipId in PipTable.StableKeys)
{
var possibleMatch = PipTable.GetPipSemiStableHash(pipId);
if (possibleMatch == SemiStableHashToRun)
{
firstMiss = new PipCacheMissInfo(pipId, PipCacheMissType.Hit);
}
}
}
Console.WriteLine("Analyzing single pip starting at: " + DateTime.Now.ToString());
AnalyzePip(firstMiss, HydratePip(firstMiss.PipId), m_writer);
}
else
{
var frontierCacheMissList = new List <PipCacheMissInfo>();
foreach (var miss in cacheMissList)
{
if (m_model.HasChangedDependencies(miss.PipId) && !AllPips)
{
continue;
}
frontierCacheMissList.Add(miss);
m_model.MarkChanged(miss.PipId);
}
Console.WriteLine("Finding frontier pips done at " + DateTime.Now.ToString());
int i = 0;
foreach (var miss in frontierCacheMissList)
{
if (i % 10 == 0)
{
Console.WriteLine("Done " + i + " of " + cacheMissList.Count);
}
var pip = HydratePip(miss.PipId);
WriteLine($"================== Analyzing pip ========================");
AnalyzePip(miss, pip, m_writer);
WriteLine("================== Complete pip ========================");
WriteLine();
i++;
}
}
return(0);
}
/// <summary> /// Consider we may get multiple cache miss info for the same pip id when cache lookup fails/timeouts and it is retried /// </summary> internal void AddCacheMiss(PipCacheMissInfo cacheMissInfo) => m_pipCacheMissesDict[cacheMissInfo.PipId] = cacheMissInfo;
