public void EverythingAsync()
{
const int MaxExecTime = 24 * 3600 * 1000;
var stream = new MemoryStream();
var r = new Random(0);
for (int i = 0; i < 10; i++)
{
int seed = r.Next(100 * 1000);
HistoricPerfDataTable table = new HistoricPerfDataTable(LoggingContext);
XAssert.IsTrue(table.Count == 0);
var s = new Random(seed);
var buffer = new byte[sizeof(long)];
for (int j = 0; j < 10; j++)
{
s.NextBytes(buffer);
long semiStableHash = BitConverter.ToInt64(buffer, 0);
var execTime = (uint)s.Next(MaxExecTime);
var processPipExecutionPerformance = new ProcessPipExecutionPerformance(
PipExecutionLevel.Executed,
DateTime.UtcNow,
DateTime.UtcNow.AddMilliseconds(execTime),
FingerprintUtilities.ZeroFingerprint,
TimeSpan.FromMilliseconds(execTime),
default(FileMonitoringViolationCounters),
default(IOCounters),
TimeSpan.FromMilliseconds(execTime),
TimeSpan.FromMilliseconds(execTime / 2),
ProcessMemoryCounters.CreateFromMb(1024, 1024, 1024, 1024),
1,
workerId: 0,
suspendedDurationMs: 0);
ProcessPipHistoricPerfData runTimeData = new ProcessPipHistoricPerfData(processPipExecutionPerformance, execTime);
table[semiStableHash] = runTimeData;
}
XAssert.IsTrue(table.Count == 10);
stream.Position = 0;
table.Save(stream);
stream.Position = 0;
table = HistoricPerfDataTable.Load(LoggingContext, stream);
XAssert.IsTrue(table.Count == 10);
s = new Random(seed);
for (int j = 0; j < 10; j++)
{
s.NextBytes(buffer);
long semiStableHash = BitConverter.ToInt64(buffer, 0);
XAssert.IsTrue(table[semiStableHash].ExeDurationInMs >= (uint)s.Next(MaxExecTime));
}
}
}
/// <summary>
/// Construct a new runtime data based on collected performance data
/// </summary>
public PipHistoricPerfData(ProcessPipExecutionPerformance executionPerformance)
{
Contract.Requires(executionPerformance.ExecutionLevel == PipExecutionLevel.Executed);
m_entryTimeToLive = DefaultTimeToLive;
DurationInMs = (uint)Math.Min(uint.MaxValue, Math.Max(1, executionPerformance.ProcessExecutionTime.TotalMilliseconds));
// For historical ram usage, we record the peak working set instead of the virtual memory due to the precision.
MemoryCounters = executionPerformance.MemoryCounters;
DiskIOInKB = (uint)Math.Min(uint.MaxValue, executionPerformance.IO.GetAggregateIO().TransferCount / 1024);
ProcessorsInPercents = executionPerformance.ProcessorsInPercents;
}
/// <inheritdoc />
public override void PipExecutionPerformance(PipExecutionPerformanceEventData data)
{
if (data.ExecutionPerformance != null)
{
ProcessPipExecutionPerformance processPerformance = data.ExecutionPerformance as ProcessPipExecutionPerformance;
if (processPerformance != null && !m_processPerformance.ContainsKey(data.PipId))
{
m_processPerformance.Add(data.PipId, processPerformance);
}
}
}
/// <summary>
/// Construct a new runtime data based on collected performance data
/// </summary>
public ProcessPipHistoricPerfData(ProcessPipExecutionPerformance executionPerformance)
{
Contract.Requires(executionPerformance.ExecutionLevel == PipExecutionLevel.Executed);
m_entryTimeToLive = DefaultTimeToLive;
// Deduct the suspended duration from the process execution time.
DurationInMs = (uint)Math.Min(uint.MaxValue, Math.Max(1, executionPerformance.ProcessExecutionTime.TotalMilliseconds - executionPerformance.SuspendedDurationMs));
// For historical ram usage, we record the peak working set instead of the virtual memory due to the precision.
MemoryCounters = executionPerformance.MemoryCounters;
DiskIOInMB = (uint)Math.Min(uint.MaxValue, ByteSizeFormatter.ToMegabytes(executionPerformance.IO.GetAggregateIO().TransferCount));
ProcessorsInPercents = executionPerformance.ProcessorsInPercents;
}
/// <summary>
/// Construct a new runtime data based on collected performance data
/// </summary>
public PipHistoricPerfData(ProcessPipExecutionPerformance executionPerformance)
{
Contract.Requires(executionPerformance.ExecutionLevel == PipExecutionLevel.Executed);
m_entryTimeToLive = DefaultTimeToLive;
DurationInMs = (uint)Math.Min(uint.MaxValue, Math.Max(1, executionPerformance.ProcessExecutionTime.TotalMilliseconds));
PeakMemoryInKB = (uint)Math.Min(uint.MaxValue, executionPerformance.PeakMemoryUsage / 1024);
DiskIOInKB = (uint)Math.Min(uint.MaxValue, executionPerformance.IO.GetAggregateIO().TransferCount / 1024);
double cpuTime = executionPerformance.KernelTime.TotalMilliseconds + executionPerformance.UserTime.TotalMilliseconds;
double processorPercentage = DurationInMs == 0 ? 0 : cpuTime / DurationInMs;
ProcessorsInPercents = (ushort)Math.Min(ushort.MaxValue, processorPercentage * 100.0);
}
/// <inheritdoc />
public override void PipExecutionPerformance(PipExecutionPerformanceEventData data)
{
if (data.ExecutionPerformance != null)
{
ProcessPipExecutionPerformance processPerformance = data.ExecutionPerformance as ProcessPipExecutionPerformance;
if (processPerformance != null)
{
var times = m_elapsedTimes[data.PipId.Value];
times.KernalTime = processPerformance.KernelTime;
times.UserTime = processPerformance.UserTime;
m_elapsedTimes[data.PipId.Value] = times;
}
}
}
/// <inheritdoc />
public override void PipExecutionPerformance(PipExecutionPerformanceEventData data)
{
ProcessPipExecutionPerformance performance = data.ExecutionPerformance as ProcessPipExecutionPerformance;
if (performance != null)
{
if (performance.FileMonitoringViolations.HasUncacheableFileAccesses)
{
UncacheablePips.Add(data.PipId);
}
if (performance.FileMonitoringViolations.NumFileAccessViolationsNotWhitelisted > 0)
{
// Non-whitelisted pips that have file access violations are not cached.
// This can occur in a passing build if UnexpectedFileAccessesAreErrors is disabled.
UncacheablePips.Add(data.PipId);
}
}
}
public void TimeToLive()
{
int execTime = 1;
uint runTime = 2;
var processPipExecutionPerformance = new ProcessPipExecutionPerformance(
PipExecutionLevel.Executed,
DateTime.UtcNow,
DateTime.UtcNow.AddMilliseconds(execTime),
FingerprintUtilities.ZeroFingerprint,
TimeSpan.FromMilliseconds(execTime),
default(FileMonitoringViolationCounters),
default(IOCounters),
TimeSpan.FromMilliseconds(execTime),
TimeSpan.FromMilliseconds(execTime / 2),
ProcessMemoryCounters.CreateFromMb(1024, 1024, 1024, 1024),
1,
workerId: 0,
suspendedDurationMs: 0);
ProcessPipHistoricPerfData runTimeData = new ProcessPipHistoricPerfData(processPipExecutionPerformance, runTime);
HistoricPerfDataTable table = new HistoricPerfDataTable(LoggingContext);
var semiStableHashToKeep = 0;
table[semiStableHashToKeep] = runTimeData;
var semiStableHashToDrop = 1;
table[semiStableHashToDrop] = runTimeData;
var stream = new MemoryStream();
for (int i = 0; i < ProcessPipHistoricPerfData.DefaultTimeToLive; i++)
{
stream.Position = 0;
table.Save(stream);
stream.Position = 0;
table = HistoricPerfDataTable.Load(LoggingContext, stream);
Analysis.IgnoreResult(table[semiStableHashToKeep]);
}
stream.Position = 0;
table = HistoricPerfDataTable.Load(LoggingContext, stream);
XAssert.AreEqual(1u, table[semiStableHashToKeep].ExeDurationInMs);
XAssert.AreEqual(0u, table[semiStableHashToDrop].ExeDurationInMs);
}
public void PipHistoricPerfDataConstructorDoesntCrash()
{
foreach (var obj in ConstructorDoesntCrashTestData())
{
var executionStart = (DateTime)obj[0];
var executionStop = (DateTime)obj[1];
var processExecutionTime = (TimeSpan)obj[2];
var fileMonitoringWarnings = (int)obj[3];
var ioCounters = (IOCounters)obj[4];
var userTime = (TimeSpan)obj[5];
var kernelTime = (TimeSpan)obj[6];
var peakMemoryUsage = (ulong)obj[7];
var numberOfProcesses = (uint)obj[8];
var workerId = (uint)obj[9];
if (executionStart > executionStop)
{
continue;
}
var performance = new ProcessPipExecutionPerformance(
PipExecutionLevel.Executed,
executionStart,
executionStop,
FingerprintUtilities.ZeroFingerprint,
processExecutionTime,
new FileMonitoringViolationCounters(fileMonitoringWarnings, fileMonitoringWarnings, fileMonitoringWarnings),
ioCounters,
userTime,
kernelTime,
ProcessMemoryCounters.CreateFromBytes(peakMemoryUsage, peakMemoryUsage, peakMemoryUsage, peakMemoryUsage),
numberOfProcesses,
workerId,
0);
var data = new ProcessPipHistoricPerfData(performance, (long)processExecutionTime.TotalMilliseconds);
data = data.Merge(data);
Analysis.IgnoreResult(data);
}
}
/// <summary>
/// Deserialize result from reader
/// </summary>
public ExecutionResult Deserialize(BuildXLReader reader, uint workerId)
{
int minAbsolutePathValue = reader.ReadInt32();
Contract.Assert(
minAbsolutePathValue == m_maxSerializableAbsolutePathIndex,
"When deserializing for distribution, the minimum absolute path value must match the serialized minimum absolute path value");
var result = (PipResultStatus)reader.ReadByte();
var converged = reader.ReadBoolean();
var numberOfWarnings = reader.ReadInt32Compact();
var weakFingerprint = reader.ReadNullableStruct(r => r.ReadWeakFingerprint());
ProcessPipExecutionPerformance performanceInformation;
if (reader.ReadBoolean())
{
performanceInformation = ProcessPipExecutionPerformance.Deserialize(reader);
// TODO: It looks like this is the wrong class for WorkerId, because the serialized object has always WorkerId == 0.
performanceInformation.WorkerId = workerId;
}
else
{
performanceInformation = null;
}
var outputContent = ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .FromWithoutCopy(ReadOutputContent(reader));
var directoryOutputs = ReadOnlyArray <(DirectoryArtifact, ReadOnlyArray <FileArtifact>)> .FromWithoutCopy(ReadDirectoryOutputs(reader));
var mustBeConsideredPerpetuallyDirty = reader.ReadBoolean();
var dynamicallyObservedFiles = reader.ReadReadOnlyArray(ReadAbsolutePath);
var dynamicallyProbedFiles = reader.ReadReadOnlyArray(ReadAbsolutePath);
var dynamicallyObservedEnumerations = reader.ReadReadOnlyArray(ReadAbsolutePath);
var allowedUndeclaredSourceReads = reader.ReadReadOnlySet(ReadAbsolutePath);
var absentPathProbesUnderOutputDirectories = reader.ReadReadOnlySet(ReadAbsolutePath);
ReportedFileAccess[] fileAccessViolationsNotWhitelisted;
ReportedFileAccess[] whitelistedFileAccessViolations;
ReportedProcess[] reportedProcesses;
ReadReportedProcessesAndFileAccesses(reader, out fileAccessViolationsNotWhitelisted, out whitelistedFileAccessViolations, out reportedProcesses, readPath: m_readPath);
var twoPhaseCachingInfo = ReadTwoPhaseCachingInfo(reader);
var cacheDescriptor = ReadPipCacheDescriptor(reader);
if (cacheDescriptor != null)
{
cacheDescriptor.OutputContentReplicasMiniBloomFilter = reader.ReadUInt32();
}
ObservedPathSet?pathSet = null;
if (reader.ReadBoolean())
{
var maybePathSet = ObservedPathSet.TryDeserialize(m_executionContext.PathTable, reader, pathReader: ReadAbsolutePath);
pathSet = maybePathSet.Succeeded
? (ObservedPathSet?)maybePathSet.Result
: null;
}
CacheLookupPerfInfo cacheLookupCounters = null;
if (reader.ReadBoolean())
{
cacheLookupCounters = CacheLookupPerfInfo.Deserialize(reader);
}
if (!result.IndicatesNoOutput())
{
// Successful result needs to be changed to not materialized because
// the process outputs are not materialized on the local machine.
result = PipResultStatus.NotMaterialized;
}
var pipProperties = ReadPipProperties(reader);
var hasUserRetries = reader.ReadBoolean();
var processExecutionResult = ExecutionResult.CreateSealed(
result,
numberOfWarnings,
outputContent,
directoryOutputs,
performanceInformation,
weakFingerprint,
fileAccessViolationsNotWhitelisted,
whitelistedFileAccessViolations,
mustBeConsideredPerpetuallyDirty,
dynamicallyObservedFiles,
dynamicallyProbedFiles,
dynamicallyObservedEnumerations,
allowedUndeclaredSourceReads,
absentPathProbesUnderOutputDirectories,
twoPhaseCachingInfo,
cacheDescriptor,
converged,
pathSet,
cacheLookupCounters,
pipProperties,
hasUserRetries);
return(processExecutionResult);
}
public void PipHistoricPerfDataConstructorDoesntCrash()
{
// TODO: Use IntelliTest/Pex for this.
var times = new[]
{
new DateTime(DateTime.MinValue.Year + 1, 1, 1).ToUniversalTime(),
new DateTime(2015, 1, 1).ToUniversalTime(),
new DateTime(DateTime.MaxValue.Year, 1, 1).ToUniversalTime()
};
var spans = new[] { TimeSpan.Zero, TimeSpan.FromSeconds(1), TimeSpan.MaxValue };
var ints = new[] { 0, int.MaxValue };
var ulongs = new ulong[] { 0, 1, ulong.MaxValue };
var uints = new uint[] { 0, 1, uint.MaxValue };
foreach (var executionStart in times)
{
foreach (var executionStop in times)
{
foreach (var processExecutionTime in spans)
{
foreach (var fileMonitoringWarnings in ints)
{
foreach (var ioCounters in new[]
{
new IOCounters(
readCounters: new IOTypeCounters(operationCount: 1, transferCount: ulong.MaxValue),
writeCounters: new IOTypeCounters(operationCount: 0, transferCount: 0),
otherCounters: new IOTypeCounters(operationCount: 0, transferCount: 0)
),
new IOCounters(
readCounters: new IOTypeCounters(operationCount: 0, transferCount: 0),
writeCounters: new IOTypeCounters(operationCount: 0, transferCount: 0),
otherCounters: new IOTypeCounters(operationCount: 0, transferCount: 0)
)
})
{
foreach (var userTime in spans)
{
foreach (var kernelTime in spans)
{
foreach (var peakMemoryUsage in ulongs)
{
foreach (var numberOfProcesses in uints)
{
foreach (var workerId in uints)
{
if (executionStart > executionStop)
{
continue;
}
var performance = new ProcessPipExecutionPerformance(
PipExecutionLevel.Executed,
executionStart,
executionStop,
FingerprintUtilities.ZeroFingerprint,
processExecutionTime,
new FileMonitoringViolationCounters(fileMonitoringWarnings, fileMonitoringWarnings, fileMonitoringWarnings),
ioCounters,
userTime,
kernelTime,
peakMemoryUsage,
numberOfProcesses,
workerId);
var data = new PipHistoricPerfData(performance);
data = data.Merge(data);
Analysis.IgnoreResult(data);
}
}
}
}
}
}
}
}
}
}
}
