private RuntimeCacheMissAnalyzer(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
FingerprintStore previousFingerprintStore,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
CacheMissDiffFormat cacheMissDiffFormat,
bool cacheMissBatch,
FingerprintStoreTestHooks testHooks = null)
{
m_loggingContext = loggingContext;
m_logTarget = logTarget;
m_context = context;
PreviousFingerprintStore = previousFingerprintStore;
m_visitor = new NodeVisitor(graph);
m_changedPips = new VisitationTracker(graph);
m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>();
m_runnablePipPerformance = runnablePipPerformance;
m_cacheMissDiffFormat = cacheMissDiffFormat;
m_maxCacheMissCanPerform = cacheMissBatch ? EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value * EngineEnvironmentSettings.MaxMessagesPerBatch : EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value;
m_batchLoggingQueue = cacheMissBatch ? NagleQueue <JProperty> .Create(
BatchLogging,
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(1),
batchSize : EngineEnvironmentSettings.MaxMessagesPerBatch) : null;
m_testHooks = testHooks;
m_testHooks?.InitRuntimeCacheMisses();
}
private RuntimeCacheMissAnalyzer(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
FingerprintStore previousFingerprintStore,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
IConfiguration configuration,
string downLoadedPreviousFingerprintStoreSavedPath,
FingerprintStoreTestHooks testHooks = null)
{
m_loggingContext = loggingContext;
m_logTarget = logTarget;
m_context = context;
PreviousFingerprintStore = previousFingerprintStore;
m_visitor = new NodeVisitor(graph);
m_changedPips = new VisitationTracker(graph);
m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>();
m_runnablePipPerformance = runnablePipPerformance;
m_batchLoggingQueue = configuration.Logging.CacheMissBatch ? NagleQueue <JProperty> .Create(
BatchLogging,
maxDegreeOfParallelism : 1,
interval : TimeSpan.FromMinutes(5),
batchSize : 100) : null;
m_testHooks = testHooks;
m_testHooks?.InitRuntimeCacheMisses();
m_configuration = configuration;
m_downLoadedPreviousFingerprintStoreSavedPath = downLoadedPreviousFingerprintStoreSavedPath;
}
private RuntimeCacheMissAnalyzer(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
FingerprintStore previousFingerprintStore,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
m_loggingContext = loggingContext;
m_logTarget = logTarget;
m_context = context;
PreviousFingerprintStore = previousFingerprintStore;
m_visitor = new NodeVisitor(graph);
m_changedPips = new VisitationTracker(graph);
m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>();
m_runnablePipPerformance = runnablePipPerformance;
}
/// <summary>
/// Initiates the task to load the fingerprint store that will be used for cache miss analysis
/// </summary>
public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration))
{
var option = configuration.Logging.CacheMissAnalysisOption;
if (option.Mode == CacheMissMode.Disabled)
{
return(null);
}
Possible <FingerprintStore> possibleStore;
if (option.Mode == CacheMissMode.Local)
{
possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext);
}
else
{
string path = null;
if (option.Mode == CacheMissMode.CustomPath)
{
path = option.CustomPath.ToString(context.PathTable);
}
else
{
Contract.Assert(option.Mode == CacheMissMode.Remote);
foreach (var key in option.Keys)
{
var cacheSavePath = configuration.Logging.FingerprintsLogDirectory
.Combine(context.PathTable, Scheduler.FingerprintStoreDirectory + "." + key);
#pragma warning disable AsyncFixer02 // This should explicitly happen synchronously since it interacts with the PathTable and StringTable
var result = cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, context.PathTable, key, configuration.Schedule.EnvironmentFingerprint).Result;
#pragma warning restore AsyncFixer02
if (result.Succeeded && result.Result)
{
path = cacheSavePath.ToString(context.PathTable);
break;
}
}
if (string.IsNullOrEmpty(path))
{
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}"));
return(null);
}
}
// Unblock caller
// WARNING: The rest can simultenously happen with saving the graph files to disk.
// We should not create any paths or strings by using PathTable and StringTable.
await Task.Yield();
possibleStore = FingerprintStore.Open(path, readOnly: true);
}
if (possibleStore.Succeeded)
{
Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory);
return(new RuntimeCacheMissAnalyzer(logTarget, loggingContext, context, possibleStore.Result, graph, runnablePipPerformance));
}
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}"));
return(null);
}
}
/// <summary>
/// Initiates the task to load the fingerprint store that will be used for cache miss analysis
/// </summary>
public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync(
FingerprintStoreExecutionLogTarget logTarget,
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance,
FingerprintStoreTestHooks testHooks = null)
{
// Unblock caller
await Task.Yield();
using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration))
{
var option = configuration.Logging.CacheMissAnalysisOption;
string downLoadedPriviousFingerprintStoreSavedPath = null;
if (option.Mode == CacheMissMode.Disabled)
{
return(null);
}
Possible <FingerprintStore> possibleStore;
PathTable newPathTable = new PathTable();
if (option.Mode == CacheMissMode.Local)
{
possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext);
}
else
{
string path = null;
if (option.Mode == CacheMissMode.CustomPath)
{
path = option.CustomPath.ToString(context.PathTable);
}
else
{
Contract.Assert(option.Mode == CacheMissMode.Remote);
foreach (var key in option.Keys)
{
var fingerprintsLogDirectoryStr = configuration.Logging.FingerprintsLogDirectory.ToString(context.PathTable);
var fingerprintsLogDirectory = AbsolutePath.Create(newPathTable, fingerprintsLogDirectoryStr);
var cacheSavePath = fingerprintsLogDirectory.Combine(newPathTable, Scheduler.FingerprintStoreDirectory + "." + key);
var result = await cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, newPathTable, key, configuration, context.CancellationToken);
if (result.Succeeded && result.Result)
{
path = cacheSavePath.ToString(newPathTable);
downLoadedPriviousFingerprintStoreSavedPath = path;
break;
}
}
if (string.IsNullOrEmpty(path))
{
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}"));
return(null);
}
}
possibleStore = FingerprintStore.Open(path, readOnly: true);
}
if (possibleStore.Succeeded)
{
Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory);
return(new RuntimeCacheMissAnalyzer(
logTarget,
loggingContext,
context,
possibleStore.Result,
graph,
runnablePipPerformance,
configuration,
downLoadedPriviousFingerprintStoreSavedPath,
testHooks: testHooks));
}
Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}"));
return(null);
}
}