private RuntimeCacheMissAnalyzer( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, FingerprintStore previousFingerprintStore, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance, CacheMissDiffFormat cacheMissDiffFormat, bool cacheMissBatch, FingerprintStoreTestHooks testHooks = null) { m_loggingContext = loggingContext; m_logTarget = logTarget; m_context = context; PreviousFingerprintStore = previousFingerprintStore; m_visitor = new NodeVisitor(graph); m_changedPips = new VisitationTracker(graph); m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>(); m_runnablePipPerformance = runnablePipPerformance; m_cacheMissDiffFormat = cacheMissDiffFormat; m_maxCacheMissCanPerform = cacheMissBatch ? EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value * EngineEnvironmentSettings.MaxMessagesPerBatch : EngineEnvironmentSettings.MaxNumPipsForCacheMissAnalysis.Value; m_batchLoggingQueue = cacheMissBatch ? NagleQueue <JProperty> .Create( BatchLogging, maxDegreeOfParallelism : 1, interval : TimeSpan.FromMinutes(1), batchSize : EngineEnvironmentSettings.MaxMessagesPerBatch) : null; m_testHooks = testHooks; m_testHooks?.InitRuntimeCacheMisses(); }
private RuntimeCacheMissAnalyzer( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, FingerprintStore previousFingerprintStore, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance, IConfiguration configuration, string downLoadedPreviousFingerprintStoreSavedPath, FingerprintStoreTestHooks testHooks = null) { m_loggingContext = loggingContext; m_logTarget = logTarget; m_context = context; PreviousFingerprintStore = previousFingerprintStore; m_visitor = new NodeVisitor(graph); m_changedPips = new VisitationTracker(graph); m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>(); m_runnablePipPerformance = runnablePipPerformance; m_batchLoggingQueue = configuration.Logging.CacheMissBatch ? NagleQueue <JProperty> .Create( BatchLogging, maxDegreeOfParallelism : 1, interval : TimeSpan.FromMinutes(5), batchSize : 100) : null; m_testHooks = testHooks; m_testHooks?.InitRuntimeCacheMisses(); m_configuration = configuration; m_downLoadedPreviousFingerprintStoreSavedPath = downLoadedPreviousFingerprintStoreSavedPath; }
private RuntimeCacheMissAnalyzer( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, FingerprintStore previousFingerprintStore, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance) { m_loggingContext = loggingContext; m_logTarget = logTarget; m_context = context; PreviousFingerprintStore = previousFingerprintStore; m_visitor = new NodeVisitor(graph); m_changedPips = new VisitationTracker(graph); m_pipCacheMissesDict = new ConcurrentDictionary <PipId, PipCacheMissInfo>(); m_runnablePipPerformance = runnablePipPerformance; }
/// <summary> /// Initiates the task to load the fingerprint store that will be used for cache miss analysis /// </summary> public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance) { using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration)) { var option = configuration.Logging.CacheMissAnalysisOption; if (option.Mode == CacheMissMode.Disabled) { return(null); } Possible <FingerprintStore> possibleStore; if (option.Mode == CacheMissMode.Local) { possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext); } else { string path = null; if (option.Mode == CacheMissMode.CustomPath) { path = option.CustomPath.ToString(context.PathTable); } else { Contract.Assert(option.Mode == CacheMissMode.Remote); foreach (var key in option.Keys) { var cacheSavePath = configuration.Logging.FingerprintsLogDirectory .Combine(context.PathTable, Scheduler.FingerprintStoreDirectory + "." + key); #pragma warning disable AsyncFixer02 // This should explicitly happen synchronously since it interacts with the PathTable and StringTable var result = cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, context.PathTable, key, configuration.Schedule.EnvironmentFingerprint).Result; #pragma warning restore AsyncFixer02 if (result.Succeeded && result.Result) { path = cacheSavePath.ToString(context.PathTable); break; } } if (string.IsNullOrEmpty(path)) { Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}")); return(null); } } // Unblock caller // WARNING: The rest can simultenously happen with saving the graph files to disk. // We should not create any paths or strings by using PathTable and StringTable. await Task.Yield(); possibleStore = FingerprintStore.Open(path, readOnly: true); } if (possibleStore.Succeeded) { Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory); return(new RuntimeCacheMissAnalyzer(logTarget, loggingContext, context, possibleStore.Result, graph, runnablePipPerformance)); } Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}")); return(null); } }
/// <summary> /// Initiates the task to load the fingerprint store that will be used for cache miss analysis /// </summary> public static async Task <RuntimeCacheMissAnalyzer> TryCreateAsync( FingerprintStoreExecutionLogTarget logTarget, LoggingContext loggingContext, PipExecutionContext context, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance, FingerprintStoreTestHooks testHooks = null) { // Unblock caller await Task.Yield(); using (logTarget.Counters.StartStopwatch(FingerprintStoreCounters.InitializeCacheMissAnalysisDuration)) { var option = configuration.Logging.CacheMissAnalysisOption; string downLoadedPriviousFingerprintStoreSavedPath = null; if (option.Mode == CacheMissMode.Disabled) { return(null); } Possible <FingerprintStore> possibleStore; PathTable newPathTable = new PathTable(); if (option.Mode == CacheMissMode.Local) { possibleStore = FingerprintStore.CreateSnapshot(logTarget.ExecutionFingerprintStore, loggingContext); } else { string path = null; if (option.Mode == CacheMissMode.CustomPath) { path = option.CustomPath.ToString(context.PathTable); } else { Contract.Assert(option.Mode == CacheMissMode.Remote); foreach (var key in option.Keys) { var fingerprintsLogDirectoryStr = configuration.Logging.FingerprintsLogDirectory.ToString(context.PathTable); var fingerprintsLogDirectory = AbsolutePath.Create(newPathTable, fingerprintsLogDirectoryStr); var cacheSavePath = fingerprintsLogDirectory.Combine(newPathTable, Scheduler.FingerprintStoreDirectory + "." + key); var result = await cache.TryRetrieveFingerprintStoreAsync(loggingContext, cacheSavePath, newPathTable, key, configuration, context.CancellationToken); if (result.Succeeded && result.Result) { path = cacheSavePath.ToString(newPathTable); downLoadedPriviousFingerprintStoreSavedPath = path; break; } } if (string.IsNullOrEmpty(path)) { Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Could not find the fingerprint store for any given key: {string.Join(",", option.Keys)}")); return(null); } } possibleStore = FingerprintStore.Open(path, readOnly: true); } if (possibleStore.Succeeded) { Logger.Log.SuccessLoadFingerprintStoreToCompare(loggingContext, option.Mode.ToString(), possibleStore.Result.StoreDirectory); return(new RuntimeCacheMissAnalyzer( logTarget, loggingContext, context, possibleStore.Result, graph, runnablePipPerformance, configuration, downLoadedPriviousFingerprintStoreSavedPath, testHooks: testHooks)); } Logger.Log.GettingFingerprintStoreTrace(loggingContext, I($"Failed to read the fingerprint store to compare. Mode: {option.Mode.ToString()} Failure: {possibleStore.Failure.DescribeIncludingInnerFailures()}")); return(null); } }