public void EverythingAsync() { const int MaxExecTime = 24 * 3600 * 1000; var stream = new MemoryStream(); var r = new Random(0); for (int i = 0; i < 10; i++) { int seed = r.Next(100 * 1000); HistoricPerfDataTable table = new HistoricPerfDataTable(LoggingContext); XAssert.IsTrue(table.Count == 0); var s = new Random(seed); var buffer = new byte[sizeof(long)]; for (int j = 0; j < 10; j++) { s.NextBytes(buffer); long semiStableHash = BitConverter.ToInt64(buffer, 0); var execTime = (uint)s.Next(MaxExecTime); var processPipExecutionPerformance = new ProcessPipExecutionPerformance( PipExecutionLevel.Executed, DateTime.UtcNow, DateTime.UtcNow.AddMilliseconds(execTime), FingerprintUtilities.ZeroFingerprint, TimeSpan.FromMilliseconds(execTime), default(FileMonitoringViolationCounters), default(IOCounters), TimeSpan.FromMilliseconds(execTime), TimeSpan.FromMilliseconds(execTime / 2), ProcessMemoryCounters.CreateFromMb(1024, 1024, 1024, 1024), 1, workerId: 0, suspendedDurationMs: 0); ProcessPipHistoricPerfData runTimeData = new ProcessPipHistoricPerfData(processPipExecutionPerformance, execTime); table[semiStableHash] = runTimeData; } XAssert.IsTrue(table.Count == 10); stream.Position = 0; table.Save(stream); stream.Position = 0; table = HistoricPerfDataTable.Load(LoggingContext, stream); XAssert.IsTrue(table.Count == 10); s = new Random(seed); for (int j = 0; j < 10; j++) { s.NextBytes(buffer); long semiStableHash = BitConverter.ToInt64(buffer, 0); XAssert.IsTrue(table[semiStableHash].ExeDurationInMs >= (uint)s.Next(MaxExecTime)); } } }
/// <summary> /// Construct a new runtime data based on collected performance data /// </summary> public PipHistoricPerfData(ProcessPipExecutionPerformance executionPerformance) { Contract.Requires(executionPerformance.ExecutionLevel == PipExecutionLevel.Executed); m_entryTimeToLive = DefaultTimeToLive; DurationInMs = (uint)Math.Min(uint.MaxValue, Math.Max(1, executionPerformance.ProcessExecutionTime.TotalMilliseconds)); // For historical ram usage, we record the peak working set instead of the virtual memory due to the precision. MemoryCounters = executionPerformance.MemoryCounters; DiskIOInKB = (uint)Math.Min(uint.MaxValue, executionPerformance.IO.GetAggregateIO().TransferCount / 1024); ProcessorsInPercents = executionPerformance.ProcessorsInPercents; }
/// <inheritdoc /> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { if (data.ExecutionPerformance != null) { ProcessPipExecutionPerformance processPerformance = data.ExecutionPerformance as ProcessPipExecutionPerformance; if (processPerformance != null && !m_processPerformance.ContainsKey(data.PipId)) { m_processPerformance.Add(data.PipId, processPerformance); } } }
/// <summary> /// Construct a new runtime data based on collected performance data /// </summary> public ProcessPipHistoricPerfData(ProcessPipExecutionPerformance executionPerformance) { Contract.Requires(executionPerformance.ExecutionLevel == PipExecutionLevel.Executed); m_entryTimeToLive = DefaultTimeToLive; // Deduct the suspended duration from the process execution time. DurationInMs = (uint)Math.Min(uint.MaxValue, Math.Max(1, executionPerformance.ProcessExecutionTime.TotalMilliseconds - executionPerformance.SuspendedDurationMs)); // For historical ram usage, we record the peak working set instead of the virtual memory due to the precision. MemoryCounters = executionPerformance.MemoryCounters; DiskIOInMB = (uint)Math.Min(uint.MaxValue, ByteSizeFormatter.ToMegabytes(executionPerformance.IO.GetAggregateIO().TransferCount)); ProcessorsInPercents = executionPerformance.ProcessorsInPercents; }
/// <summary> /// Construct a new runtime data based on collected performance data /// </summary> public PipHistoricPerfData(ProcessPipExecutionPerformance executionPerformance) { Contract.Requires(executionPerformance.ExecutionLevel == PipExecutionLevel.Executed); m_entryTimeToLive = DefaultTimeToLive; DurationInMs = (uint)Math.Min(uint.MaxValue, Math.Max(1, executionPerformance.ProcessExecutionTime.TotalMilliseconds)); PeakMemoryInKB = (uint)Math.Min(uint.MaxValue, executionPerformance.PeakMemoryUsage / 1024); DiskIOInKB = (uint)Math.Min(uint.MaxValue, executionPerformance.IO.GetAggregateIO().TransferCount / 1024); double cpuTime = executionPerformance.KernelTime.TotalMilliseconds + executionPerformance.UserTime.TotalMilliseconds; double processorPercentage = DurationInMs == 0 ? 0 : cpuTime / DurationInMs; ProcessorsInPercents = (ushort)Math.Min(ushort.MaxValue, processorPercentage * 100.0); }
/// <inheritdoc /> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { if (data.ExecutionPerformance != null) { ProcessPipExecutionPerformance processPerformance = data.ExecutionPerformance as ProcessPipExecutionPerformance; if (processPerformance != null) { var times = m_elapsedTimes[data.PipId.Value]; times.KernalTime = processPerformance.KernelTime; times.UserTime = processPerformance.UserTime; m_elapsedTimes[data.PipId.Value] = times; } } }
/// <inheritdoc /> public override void PipExecutionPerformance(PipExecutionPerformanceEventData data) { ProcessPipExecutionPerformance performance = data.ExecutionPerformance as ProcessPipExecutionPerformance; if (performance != null) { if (performance.FileMonitoringViolations.HasUncacheableFileAccesses) { UncacheablePips.Add(data.PipId); } if (performance.FileMonitoringViolations.NumFileAccessViolationsNotWhitelisted > 0) { // Non-whitelisted pips that have file access violations are not cached. // This can occur in a passing build if UnexpectedFileAccessesAreErrors is disabled. UncacheablePips.Add(data.PipId); } } }
public void TimeToLive() { int execTime = 1; uint runTime = 2; var processPipExecutionPerformance = new ProcessPipExecutionPerformance( PipExecutionLevel.Executed, DateTime.UtcNow, DateTime.UtcNow.AddMilliseconds(execTime), FingerprintUtilities.ZeroFingerprint, TimeSpan.FromMilliseconds(execTime), default(FileMonitoringViolationCounters), default(IOCounters), TimeSpan.FromMilliseconds(execTime), TimeSpan.FromMilliseconds(execTime / 2), ProcessMemoryCounters.CreateFromMb(1024, 1024, 1024, 1024), 1, workerId: 0, suspendedDurationMs: 0); ProcessPipHistoricPerfData runTimeData = new ProcessPipHistoricPerfData(processPipExecutionPerformance, runTime); HistoricPerfDataTable table = new HistoricPerfDataTable(LoggingContext); var semiStableHashToKeep = 0; table[semiStableHashToKeep] = runTimeData; var semiStableHashToDrop = 1; table[semiStableHashToDrop] = runTimeData; var stream = new MemoryStream(); for (int i = 0; i < ProcessPipHistoricPerfData.DefaultTimeToLive; i++) { stream.Position = 0; table.Save(stream); stream.Position = 0; table = HistoricPerfDataTable.Load(LoggingContext, stream); Analysis.IgnoreResult(table[semiStableHashToKeep]); } stream.Position = 0; table = HistoricPerfDataTable.Load(LoggingContext, stream); XAssert.AreEqual(1u, table[semiStableHashToKeep].ExeDurationInMs); XAssert.AreEqual(0u, table[semiStableHashToDrop].ExeDurationInMs); }
public void PipHistoricPerfDataConstructorDoesntCrash() { foreach (var obj in ConstructorDoesntCrashTestData()) { var executionStart = (DateTime)obj[0]; var executionStop = (DateTime)obj[1]; var processExecutionTime = (TimeSpan)obj[2]; var fileMonitoringWarnings = (int)obj[3]; var ioCounters = (IOCounters)obj[4]; var userTime = (TimeSpan)obj[5]; var kernelTime = (TimeSpan)obj[6]; var peakMemoryUsage = (ulong)obj[7]; var numberOfProcesses = (uint)obj[8]; var workerId = (uint)obj[9]; if (executionStart > executionStop) { continue; } var performance = new ProcessPipExecutionPerformance( PipExecutionLevel.Executed, executionStart, executionStop, FingerprintUtilities.ZeroFingerprint, processExecutionTime, new FileMonitoringViolationCounters(fileMonitoringWarnings, fileMonitoringWarnings, fileMonitoringWarnings), ioCounters, userTime, kernelTime, ProcessMemoryCounters.CreateFromBytes(peakMemoryUsage, peakMemoryUsage, peakMemoryUsage, peakMemoryUsage), numberOfProcesses, workerId, 0); var data = new ProcessPipHistoricPerfData(performance, (long)processExecutionTime.TotalMilliseconds); data = data.Merge(data); Analysis.IgnoreResult(data); } }
/// <summary> /// Deserialize result from reader /// </summary> public ExecutionResult Deserialize(BuildXLReader reader, uint workerId) { int minAbsolutePathValue = reader.ReadInt32(); Contract.Assert( minAbsolutePathValue == m_maxSerializableAbsolutePathIndex, "When deserializing for distribution, the minimum absolute path value must match the serialized minimum absolute path value"); var result = (PipResultStatus)reader.ReadByte(); var converged = reader.ReadBoolean(); var numberOfWarnings = reader.ReadInt32Compact(); var weakFingerprint = reader.ReadNullableStruct(r => r.ReadWeakFingerprint()); ProcessPipExecutionPerformance performanceInformation; if (reader.ReadBoolean()) { performanceInformation = ProcessPipExecutionPerformance.Deserialize(reader); // TODO: It looks like this is the wrong class for WorkerId, because the serialized object has always WorkerId == 0. performanceInformation.WorkerId = workerId; } else { performanceInformation = null; } var outputContent = ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .FromWithoutCopy(ReadOutputContent(reader)); var directoryOutputs = ReadOnlyArray <(DirectoryArtifact, ReadOnlyArray <FileArtifact>)> .FromWithoutCopy(ReadDirectoryOutputs(reader)); var mustBeConsideredPerpetuallyDirty = reader.ReadBoolean(); var dynamicallyObservedFiles = reader.ReadReadOnlyArray(ReadAbsolutePath); var dynamicallyProbedFiles = reader.ReadReadOnlyArray(ReadAbsolutePath); var dynamicallyObservedEnumerations = reader.ReadReadOnlyArray(ReadAbsolutePath); var allowedUndeclaredSourceReads = reader.ReadReadOnlySet(ReadAbsolutePath); var absentPathProbesUnderOutputDirectories = reader.ReadReadOnlySet(ReadAbsolutePath); ReportedFileAccess[] fileAccessViolationsNotWhitelisted; ReportedFileAccess[] whitelistedFileAccessViolations; ReportedProcess[] reportedProcesses; ReadReportedProcessesAndFileAccesses(reader, out fileAccessViolationsNotWhitelisted, out whitelistedFileAccessViolations, out reportedProcesses, readPath: m_readPath); var twoPhaseCachingInfo = ReadTwoPhaseCachingInfo(reader); var cacheDescriptor = ReadPipCacheDescriptor(reader); if (cacheDescriptor != null) { cacheDescriptor.OutputContentReplicasMiniBloomFilter = reader.ReadUInt32(); } ObservedPathSet?pathSet = null; if (reader.ReadBoolean()) { var maybePathSet = ObservedPathSet.TryDeserialize(m_executionContext.PathTable, reader, pathReader: ReadAbsolutePath); pathSet = maybePathSet.Succeeded ? (ObservedPathSet?)maybePathSet.Result : null; } CacheLookupPerfInfo cacheLookupCounters = null; if (reader.ReadBoolean()) { cacheLookupCounters = CacheLookupPerfInfo.Deserialize(reader); } if (!result.IndicatesNoOutput()) { // Successful result needs to be changed to not materialized because // the process outputs are not materialized on the local machine. result = PipResultStatus.NotMaterialized; } var pipProperties = ReadPipProperties(reader); var hasUserRetries = reader.ReadBoolean(); var processExecutionResult = ExecutionResult.CreateSealed( result, numberOfWarnings, outputContent, directoryOutputs, performanceInformation, weakFingerprint, fileAccessViolationsNotWhitelisted, whitelistedFileAccessViolations, mustBeConsideredPerpetuallyDirty, dynamicallyObservedFiles, dynamicallyProbedFiles, dynamicallyObservedEnumerations, allowedUndeclaredSourceReads, absentPathProbesUnderOutputDirectories, twoPhaseCachingInfo, cacheDescriptor, converged, pathSet, cacheLookupCounters, pipProperties, hasUserRetries); return(processExecutionResult); }
public void PipHistoricPerfDataConstructorDoesntCrash() { // TODO: Use IntelliTest/Pex for this. var times = new[] { new DateTime(DateTime.MinValue.Year + 1, 1, 1).ToUniversalTime(), new DateTime(2015, 1, 1).ToUniversalTime(), new DateTime(DateTime.MaxValue.Year, 1, 1).ToUniversalTime() }; var spans = new[] { TimeSpan.Zero, TimeSpan.FromSeconds(1), TimeSpan.MaxValue }; var ints = new[] { 0, int.MaxValue }; var ulongs = new ulong[] { 0, 1, ulong.MaxValue }; var uints = new uint[] { 0, 1, uint.MaxValue }; foreach (var executionStart in times) { foreach (var executionStop in times) { foreach (var processExecutionTime in spans) { foreach (var fileMonitoringWarnings in ints) { foreach (var ioCounters in new[] { new IOCounters( readCounters: new IOTypeCounters(operationCount: 1, transferCount: ulong.MaxValue), writeCounters: new IOTypeCounters(operationCount: 0, transferCount: 0), otherCounters: new IOTypeCounters(operationCount: 0, transferCount: 0) ), new IOCounters( readCounters: new IOTypeCounters(operationCount: 0, transferCount: 0), writeCounters: new IOTypeCounters(operationCount: 0, transferCount: 0), otherCounters: new IOTypeCounters(operationCount: 0, transferCount: 0) ) }) { foreach (var userTime in spans) { foreach (var kernelTime in spans) { foreach (var peakMemoryUsage in ulongs) { foreach (var numberOfProcesses in uints) { foreach (var workerId in uints) { if (executionStart > executionStop) { continue; } var performance = new ProcessPipExecutionPerformance( PipExecutionLevel.Executed, executionStart, executionStop, FingerprintUtilities.ZeroFingerprint, processExecutionTime, new FileMonitoringViolationCounters(fileMonitoringWarnings, fileMonitoringWarnings, fileMonitoringWarnings), ioCounters, userTime, kernelTime, peakMemoryUsage, numberOfProcesses, workerId); var data = new PipHistoricPerfData(performance); data = data.Merge(data); Analysis.IgnoreResult(data); } } } } } } } } } } }