/// <summary> /// Hooks into the log target for pip execution performance data which will be called /// when a pip fails. This will then dump relevant information on the failing pip /// to a JSON file specified under <see cref="m_logPath"/>. /// The maximum number of logs generated can be specified using the /// /DumpFailedPipsLogCount parameter. /// </summary> /// <remarks> /// If an error occurs while serializing/dumping the specified pip, /// then this analyzer will be disabled for the remainder of this build and /// a warning will be logged with more details. /// </remarks> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { if (data.ExecutionPerformance.ExecutionLevel == PipExecutionLevel.Failed && !m_loggingErrorOccured) { var currentNumLogFiles = Interlocked.Increment(ref m_numLogFilesGenerated); if (currentNumLogFiles <= m_maxLogFiles) { var pip = m_pipTable.HydratePip(data.PipId, PipQueryContext.DumpPipLiteAnalyzer); ProcessExecutionMonitoringReportedEventData?dynamicData = null; if (m_shouldDumpDynamicData) { m_dynamicDataDictionary.TryRemove(data.PipId, out dynamicData); } DumpPip(pip, dynamicData, currentNumLogFiles); } } else { // If m_loggingErrorOccured is set to true, then no data is being added to the dictionary anyway, so there is no need to remove if (!m_loggingErrorOccured && m_shouldDumpDynamicData) { // The pip is not in a failed state, so it does not need to be dumped. m_dynamicDataDictionary.TryRemove(data.PipId, out _); } } }
/// <nodoc /> public static PipExecutionPerformanceEvent ToPipExecutionPerformanceEvent(this PipExecutionPerformanceEventData data) { var pipExecPerfEvent = new PipExecutionPerformanceEvent(); var pipExecPerformance = new PipExecutionPerformance(); pipExecPerformance.PipExecutionLevel = (int)data.ExecutionPerformance.ExecutionLevel; pipExecPerformance.ExecutionStart = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(data.ExecutionPerformance.ExecutionStart); pipExecPerformance.ExecutionStop = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(data.ExecutionPerformance.ExecutionStop); var processPipExecPerformance = new Xldb.Proto.ProcessPipExecutionPerformance(); var performance = data.ExecutionPerformance as ProcessPipExecutionPerformance; if (performance != null) { processPipExecPerformance.ProcessExecutionTime = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(performance.ProcessExecutionTime); processPipExecPerformance.ReadCounters = new IOTypeCounters() { OperationCount = performance.IO.ReadCounters.OperationCount, TransferCOunt = performance.IO.ReadCounters.TransferCount }; processPipExecPerformance.WriteCounters = new IOTypeCounters() { OperationCount = performance.IO.WriteCounters.OperationCount, TransferCOunt = performance.IO.WriteCounters.TransferCount }; processPipExecPerformance.OtherCounters = new IOTypeCounters() { OperationCount = performance.IO.OtherCounters.OperationCount, TransferCOunt = performance.IO.OtherCounters.TransferCount }; processPipExecPerformance.UserTime = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(performance.UserTime); processPipExecPerformance.KernelTime = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(performance.KernelTime); processPipExecPerformance.PeakMemoryUsage = performance.PeakMemoryUsage; processPipExecPerformance.PeakMemoryUsageMb = performance.PeakMemoryUsageMb; processPipExecPerformance.NumberOfProcesses = performance.NumberOfProcesses; processPipExecPerformance.FileMonitoringViolationCounters = new FileMonitoringViolationCounters() { NumFileAccessesWhitelistedAndCacheable = performance.FileMonitoringViolations.NumFileAccessesWhitelistedAndCacheable, NumFileAccessesWhitelistedButNotCacheable = performance.FileMonitoringViolations.NumFileAccessesWhitelistedButNotCacheable, NumFileAccessViolationsNotWhitelisted = performance.FileMonitoringViolations.NumFileAccessViolationsNotWhitelisted }; processPipExecPerformance.Fingerprint = performance.Fingerprint.ToFingerprint(); if (performance.CacheDescriptorId.HasValue) { processPipExecPerformance.CacheDescriptorId = performance.CacheDescriptorId.Value; } } pipExecPerfEvent.WorkerID = data.ExecutionPerformance.WorkerId; pipExecPerfEvent.PipID = data.PipId.Value; pipExecPerfEvent.PipExecutionPerformance = pipExecPerformance; pipExecPerfEvent.ProcessPipExecutionPerformance = processPipExecPerformance; return(pipExecPerfEvent); }
public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { // Got a performance event for a process so register the process as completed if (CachedGraph.PipTable.GetPipType(data.PipId) == PipType.Process) { m_completedPips.Add(data.PipId); } }
/// <inheritdoc /> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { if (data.ExecutionPerformance != null) { ProcessPipExecutionPerformance processPerformance = data.ExecutionPerformance as ProcessPipExecutionPerformance; if (processPerformance != null && !m_processPerformance.ContainsKey(data.PipId)) { m_processPerformance.Add(data.PipId, processPerformance); } } }
public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { if (data.PipId == m_pip.PipId) { m_sections.Add( m_html.CreateBlock( "Pip Execution Performance", m_html.CreateRow("Execution Start", data.ExecutionPerformance.ExecutionStart), m_html.CreateRow("Execution Stop", data.ExecutionPerformance.ExecutionStop), m_html.CreateRow("WorkerId", data.ExecutionPerformance.WorkerId.ToString(CultureInfo.InvariantCulture)), m_html.CreateEnumRow("ExecutionLevel", data.ExecutionPerformance.ExecutionLevel))); } }
/// <inheritdoc /> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { if (data.ExecutionPerformance != null) { ProcessPipExecutionPerformance processPerformance = data.ExecutionPerformance as ProcessPipExecutionPerformance; if (processPerformance != null) { var times = m_elapsedTimes[data.PipId.Value]; times.KernalTime = processPerformance.KernelTime; times.UserTime = processPerformance.UserTime; m_elapsedTimes[data.PipId.Value] = times; } } }
/// <summary> /// Override event to capture its data and store it in the protobuf /// </summary> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { var value = data.ToPipExecutionPerformanceEvent(); var key = new EventKey { EventTypeID = Xldb.Proto.ExecutionEventId.PipExecutionPerformance, PipId = data.PipId.Value }; var keyArr = key.ToByteArray(); var valueArr = value.ToByteArray(); WriteToDb(keyArr, valueArr, XldbDataStore.EventColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.PipExecutionPerformance, keyArr.Length + valueArr.Length); }
public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { var semistableHash = CachedGraph.PipTable.GetPipSemiStableHash(data.PipId); if (!IsFailedPipOrDependency(semistableHash)) { // Only record inputs for failed pips or their transitive dependencies return; } if (data.ExecutionPerformance is ProcessPipExecutionPerformance processPipPerformance) { // Save the performance event of failed Pips m_pipPerformance[data.PipId.Value] = processPipPerformance; } m_semiStableHashProcessPips.Add(semistableHash, data.PipId); }
public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { // Record set of failed pips and cached pips if (CachedGraph.PipTable.GetPipType(data.PipId) == PipType.Process) { switch (data.ExecutionPerformance.ExecutionLevel) { case PipExecutionLevel.Cached: case PipExecutionLevel.UpToDate: m_cachedPips.MarkVisited(data.PipId.ToNodeId()); break; case PipExecutionLevel.Failed: m_failedPips.Add(data.PipId); break; } } }
/// <summary> /// Hooks into the log target for pip execution performance data which will be called /// when a pip fails. This will then dump relevant information on the failing pip /// to a JSON file specified under <see cref="m_logPath"/>. /// The maximum number of logs generated can be specified using the /// /DumpFailedPipsLogCount parameter. /// </summary> /// <remarks> /// If an error occurs while serializing/dumping the specified pip, /// then this analyzer will be disabled for the remainder of this build and /// a warning will be logged with more details. /// </remarks> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { if (data.ExecutionPerformance.ExecutionLevel == PipExecutionLevel.Failed && !m_loggingErrorOccured) { var currentNumLogFiles = Interlocked.Increment(ref m_numLogFilesGenerated); if (currentNumLogFiles <= m_maxLogFiles) { var dumpPipResult = false; if (!m_logPathCreated) { // A log entry should have been generated already if this fails m_logPathCreated = DumpPipLiteAnalysisUtilities.CreateLoggingDirectory(m_logPath.ToString(m_pipExecutionContext.PathTable), m_loggingContext); } if (m_logPathCreated) { var pip = m_pipTable.HydratePip(data.PipId, PipQueryContext.DumpPipLiteAnalyzer); // A log entry should have been generated already if this fails dumpPipResult = DumpPipLiteAnalysisUtilities.DumpPip(pip, m_logPath.ToString(m_pipExecutionContext.PathTable), m_pipExecutionContext.PathTable, m_pipExecutionContext.StringTable, m_pipExecutionContext.SymbolTable, m_pipGraph); } if (!(m_logPathCreated && dumpPipResult)) { // This failure was already logged in DumpPipLiteAnalysisUtilies m_loggingErrorOccured = true; } if (currentNumLogFiles >= m_maxLogFiles) { // Log limit reached, log this once Logger.Log.RuntimeDumpPipLiteLogLimitReached(m_loggingContext, m_maxLogFiles); } } } }
/// <inheritdoc /> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { ProcessPipExecutionPerformance performance = data.ExecutionPerformance as ProcessPipExecutionPerformance; if (performance != null) { if (performance.FileMonitoringViolations.HasUncacheableFileAccesses) { UncacheablePips.Add(data.PipId); } if (performance.FileMonitoringViolations.NumFileAccessViolationsNotWhitelisted > 0) { // Non-whitelisted pips that have file access violations are not cached. // This can occur in a passing build if UnexpectedFileAccessesAreErrors is disabled. UncacheablePips.Add(data.PipId); } } }
public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { if (CachedGraph.PipTable.GetPipType(data.PipId) != PipType.Process) { // Skip other than Process return; } var executionPerformance = data.ExecutionPerformance; if (executionPerformance != null) { m_pipExecutionPerf.Add(data.PipId, executionPerformance); if (executionPerformance.ExecutionStart < m_firstPip) { m_firstPip = executionPerformance.ExecutionStart; } } }
private BxlPipData BuildBxlPipFromExecutionData( Pip pip, PipExecutionPerformanceEventData executionData, MultiValueDictionary <uint, byte> scheduledPipDependencyCount, MultiValueDictionary <uint, byte> scheduledPipInputCount, MultiValueDictionary <uint, byte> scheduledPipOutputCount) { var pipData = new BxlPipData { SemiStableHash = pip.FormattedSemiStableHash, DurationMs = (executionData.ExecutionPerformance.ExecutionStop - executionData.ExecutionPerformance.ExecutionStart).TotalMilliseconds, Type = pip.PipType.ToString(), ExecutionLevel = executionData.ExecutionPerformance.ExecutionLevel.ToString(), StartTimeTicks = executionData.ExecutionPerformance.ExecutionStart.Ticks, TagCount = pip.Tags.Count() }; if (scheduledPipDependencyCount.TryGetValue(pip.PipId.Value, out var deps)) { pipData.DependencyCount = deps.Count; } if (scheduledPipInputCount.TryGetValue(pip.PipId.Value, out var inputs)) { pipData.InputCount = inputs.Count; } if (scheduledPipOutputCount.TryGetValue(pip.PipId.Value, out var outputs)) { pipData.OutputCount = outputs.Count; } if (pip.PipType == PipType.Process) { var process = pip as Pips.Operations.Process; pipData.Priority = process.Priority; pipData.Weight = process.Weight; pipData.SemaphoreCount = process.Semaphores.Length; } // done... return(pipData); }
public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { // Record set of failed pips and cached pips // Note: Having the DXXXX error code as part of this event will help // make an early desicion of the action to take for processing. if (CachedGraph.PipTable.GetPipType(data.PipId) == PipType.Process) { switch (data.ExecutionPerformance.ExecutionLevel) { case PipExecutionLevel.Cached: case PipExecutionLevel.UpToDate: m_cachedPips.MarkVisited(data.PipId.ToNodeId()); break; case PipExecutionLevel.Failed: { m_failedPips.Add(data.PipId); if (data.ExecutionPerformance is ProcessPipExecutionPerformance processPipPerformance) { // Save the performance event of failed Pips m_pipPerformance[data.PipId.Value] = processPipPerformance; } // Keep knowledge of semistable hash to pip // Hydradting pip here is expensive so get the semistable hash fron the CachedGraph // var pip = (Process)m_pipTable.HydratePip(data.PipId, PipQueryContext.ViewerAnalyzer); var semistableHash = CachedGraph.PipTable.GetPipSemiStableHash(data.PipId); m_semiStableHashProcessPips.Add(semistableHash, data.PipId); } break; case PipExecutionLevel.Executed: break; default: throw new ArgumentOutOfRangeException(); } } }
/// <summary> /// Dumps all failing pips is the /dumpAllFailedPips flag is set. /// </summary> /// <param name="data"></param> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { if (m_dumpAllFailedPips && data.ExecutionPerformance.ExecutionLevel == PipExecutionLevel.Failed) { if (!m_isLogDirectoryCreated) { m_isLogDirectoryCreated = DumpPipLiteAnalysisUtilities.CreateLoggingDirectory(m_logPath, LoggingContext); } if (m_isLogDirectoryCreated) { var pip = m_pipTable.HydratePip(data.PipId, PipQueryContext.DumpPipLiteAnalyzer); // A log entry should have been generated already if this fails DumpPipLiteAnalysisUtilities.DumpPip(pip, m_logPath, PathTable, StringTable, SymbolTable, m_pipGraph); } } }
/// <inheritdoc /> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { m_writeExecutionEntries.Add(data); m_criticalPathAnalyzer.PipExecutionPerformance(data); }
/// <inheritdoc /> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { m_scheduledPipExecutionData.Add(data.PipId.Value, data); }
/// <inheritdoc /> public virtual void PipExecutionPerformance(PipExecutionPerformanceEventData data) { ReportUnhandledEvent(data); }
/// <inheritdoc /> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { m_writeExecutionEntries.Add(data); }
public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { m_executedPipsTracker.MarkVisited(data.PipId.ToNodeId()); }