// <inheritdoc />
internal override JobObject.AccountingInformation GetJobAccountingInfo()
{
if (m_processResourceUsage is null)
{
return(base.GetJobAccountingInfo());
}
else
{
IOCounters ioCounters;
ProcessMemoryCounters memoryCounters;
uint childProcesses = 0;
try
{
ioCounters = new IOCounters(new IO_COUNTERS()
{
ReadOperationCount = m_processResourceUsage.Aggregate(0UL, (acc, usage) => acc + usage.Value.DiskReadOps),
ReadTransferCount = m_processResourceUsage.Aggregate(0UL, (acc, usage) => acc + usage.Value.DiskBytesRead),
WriteOperationCount = m_processResourceUsage.Aggregate(0UL, (acc, usage) => acc + usage.Value.DiskWriteOps),
WriteTransferCount = m_processResourceUsage.Aggregate(0UL, (acc, usage) => acc + usage.Value.DiskBytesWritten),
});
memoryCounters = ProcessMemoryCounters.CreateFromBytes(
m_processResourceUsage.Aggregate(0UL, (acc, usage) => acc + usage.Value.PeakWorkingSetSize),
m_processResourceUsage.Aggregate(0UL, (acc, usage) => acc + usage.Value.WorkingSetSize),
0, 0);
childProcesses = m_processResourceUsage.Keys.Count > 0 ? (uint)(m_processResourceUsage.Keys.Count - 1) : 0; // Exclude the root process from the child count
}
catch (OverflowException ex)
{
LogProcessState($"Overflow exception caught while calculating AccountingInformation:{Environment.NewLine}{ex.ToString()}");
ioCounters = new IOCounters(new IO_COUNTERS()
{
ReadOperationCount = 0,
WriteOperationCount = 0,
ReadTransferCount = 0,
WriteTransferCount = 0
});
memoryCounters = ProcessMemoryCounters.CreateFromBytes(0, 0, 0, 0);
}
return(new JobObject.AccountingInformation
{
IO = ioCounters,
MemoryCounters = memoryCounters,
KernelTime = TimeSpan.FromMilliseconds(m_processResourceUsage.Aggregate(0.0, (acc, usage) => acc + usage.Value.SystemTimeNs) / NanosecondsToMillisecondsFactor),
UserTime = TimeSpan.FromMilliseconds(m_processResourceUsage.Aggregate(0.0, (acc, usage) => acc + usage.Value.UserTimeNs) / NanosecondsToMillisecondsFactor),
NumberOfProcesses = childProcesses,
});
}
}
public void PipHistoricPerfDataConstructorDoesntCrash()
{
foreach (var obj in ConstructorDoesntCrashTestData())
{
var executionStart = (DateTime)obj[0];
var executionStop = (DateTime)obj[1];
var processExecutionTime = (TimeSpan)obj[2];
var fileMonitoringWarnings = (int)obj[3];
var ioCounters = (IOCounters)obj[4];
var userTime = (TimeSpan)obj[5];
var kernelTime = (TimeSpan)obj[6];
var peakMemoryUsage = (ulong)obj[7];
var numberOfProcesses = (uint)obj[8];
var workerId = (uint)obj[9];
if (executionStart > executionStop)
{
continue;
}
var performance = new ProcessPipExecutionPerformance(
PipExecutionLevel.Executed,
executionStart,
executionStop,
FingerprintUtilities.ZeroFingerprint,
processExecutionTime,
new FileMonitoringViolationCounters(fileMonitoringWarnings, fileMonitoringWarnings, fileMonitoringWarnings),
ioCounters,
userTime,
kernelTime,
ProcessMemoryCounters.CreateFromBytes(peakMemoryUsage, peakMemoryUsage, peakMemoryUsage, peakMemoryUsage),
numberOfProcesses,
workerId,
0);
var data = new ProcessPipHistoricPerfData(performance, (long)processExecutionTime.TotalMilliseconds);
data = data.Merge(data);
Analysis.IgnoreResult(data);
}
}
public void TestProcessExecutionResultSerialization()
{
var reportedAccess = CreateRandomReportedFileAccess();
Fingerprint fingerprint = FingerprintUtilities.CreateRandom();
var processExecutionResult = ExecutionResult.CreateSealed(
result: PipResultStatus.Succeeded,
numberOfWarnings: 12,
outputContent: ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .FromWithoutCopy(CreateRandomOutputContent(), CreateRandomOutputContent()),
directoryOutputs: ReadOnlyArray <(DirectoryArtifact, ReadOnlyArray <FileArtifact>)> .FromWithoutCopy(CreateRandomOutputDirectory(), CreateRandomOutputDirectory()),
performanceInformation: new ProcessPipExecutionPerformance(
PipExecutionLevel.Executed,
DateTime.UtcNow,
DateTime.UtcNow + TimeSpan.FromMinutes(2),
FingerprintUtilities.ZeroFingerprint,
TimeSpan.FromMinutes(2),
new FileMonitoringViolationCounters(2, 3, 4),
default(IOCounters),
TimeSpan.FromMinutes(3),
TimeSpan.FromMinutes(3),
ProcessMemoryCounters.CreateFromBytes(12324, 12325, 12326, 12326),
33,
7,
0),
fingerprint: new WeakContentFingerprint(fingerprint),
fileAccessViolationsNotAllowlisted: new[]
{
reportedAccess,
CreateRandomReportedFileAccess(),
// Create reported file access that uses the same process to test deduplication during deserialization
CreateRandomReportedFileAccess(reportedAccess.Process),
},
allowlistedFileAccessViolations: new ReportedFileAccess[0],
mustBeConsideredPerpetuallyDirty: true,
dynamicallyObservedFiles: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(
CreateSourceFile().Path,
CreateSourceFile().Path
),
dynamicallyProbedFiles: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(
CreateSourceFile().Path,
CreateSourceFile().Path,
CreateSourceFile().Path
),
dynamicallyObservedEnumerations: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(
CreateSourceFile().Path
),
allowedUndeclaredSourceReads: new ReadOnlyHashSet <AbsolutePath> {
CreateSourceFile().Path,
CreateSourceFile().Path
},
absentPathProbesUnderOutputDirectories: new ReadOnlyHashSet <AbsolutePath> {
CreateSourceFile().Path,
CreateSourceFile().Path
},
twoPhaseCachingInfo: new TwoPhaseCachingInfo(
new WeakContentFingerprint(Fingerprint.Random(FingerprintUtilities.FingerprintLength)),
ContentHashingUtilities.CreateRandom(),
new StrongContentFingerprint(Fingerprint.Random(FingerprintUtilities.FingerprintLength)),
new CacheEntry(ContentHashingUtilities.CreateRandom(), null, CreateRandomContentHashArray())),
pipCacheDescriptorV2Metadata: null,
converged: true,
pathSet: null,
cacheLookupStepDurations: null,
pipProperties: new Dictionary <string, int> {
{ "Foo", 1 }, { "Bar", 9 }
},
hasUserRetries: true);
ExecutionResultSerializer serializer = new ExecutionResultSerializer(0, Context);
ExecutionResult deserializedProcessExecutionResult;
using (var stream = new MemoryStream())
using (var writer = new BuildXLWriter(false, stream, true, false))
using (var reader = new BuildXLReader(false, stream, true))
{
serializer.Serialize(writer, processExecutionResult, preservePathCasing: false);
stream.Position = 0;
deserializedProcessExecutionResult = serializer.Deserialize(reader,
processExecutionResult.PerformanceInformation.WorkerId);
}
// Ensure successful pip result is changed to not materialized.
XAssert.AreEqual(PipResultStatus.NotMaterialized, deserializedProcessExecutionResult.Result);
AssertEqual(processExecutionResult, deserializedProcessExecutionResult,
r => r.NumberOfWarnings,
r => r.Converged,
r => r.OutputContent.Length,
r => r.DirectoryOutputs.Length,
r => r.PerformanceInformation.ExecutionLevel,
r => r.PerformanceInformation.ExecutionStop,
r => r.PerformanceInformation.ExecutionStart,
r => r.PerformanceInformation.ProcessExecutionTime,
r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessViolationsNotAllowlisted,
r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessesAllowlistedAndCacheable,
r => r.PerformanceInformation.FileMonitoringViolations.NumFileAccessesAllowlistedButNotCacheable,
r => r.PerformanceInformation.UserTime,
r => r.PerformanceInformation.KernelTime,
r => r.PerformanceInformation.MemoryCounters.PeakWorkingSetMb,
r => r.PerformanceInformation.MemoryCounters.AverageWorkingSetMb,
r => r.PerformanceInformation.MemoryCounters.PeakCommitSizeMb,
r => r.PerformanceInformation.MemoryCounters.AverageCommitSizeMb,
r => r.PerformanceInformation.NumberOfProcesses,
r => r.FileAccessViolationsNotAllowlisted.Count,
r => r.MustBeConsideredPerpetuallyDirty,
r => r.DynamicallyObservedFiles.Length,
r => r.DynamicallyProbedFiles.Length,
r => r.DynamicallyObservedEnumerations.Length,
r => r.AllowedUndeclaredReads.Count,
r => r.TwoPhaseCachingInfo.WeakFingerprint,
r => r.TwoPhaseCachingInfo.StrongFingerprint,
r => r.TwoPhaseCachingInfo.PathSetHash,
r => r.TwoPhaseCachingInfo.CacheEntry.MetadataHash,
r => r.TwoPhaseCachingInfo.CacheEntry.OriginatingCache,
r => r.TwoPhaseCachingInfo.CacheEntry.ReferencedContent.Length,
r => r.PipProperties.Count,
r => r.HasUserRetries,
r => r.RetryInfo
);
for (int i = 0; i < processExecutionResult.OutputContent.Length; i++)
{
int j = i;
AssertEqual(processExecutionResult, deserializedProcessExecutionResult,
r => r.OutputContent[j].Item1,
r => r.OutputContent[j].Item2
);
// Ensure that output origin from deserialzed output content is changed to not materialized.
XAssert.AreEqual(PipOutputOrigin.NotMaterialized, deserializedProcessExecutionResult.OutputContent[i].Item3);
}
for (int i = 0; i < processExecutionResult.DirectoryOutputs.Length; i++)
{
var expected = processExecutionResult.DirectoryOutputs[i];
var result = deserializedProcessExecutionResult.DirectoryOutputs[i];
XAssert.AreEqual(expected.Item1, result.Item1);
XAssert.AreEqual(expected.Item2.Length, result.Item2.Length);
for (int j = 0; j < expected.Item2.Length; j++)
{
XAssert.AreEqual(expected.Item2[j], result.Item2[j]);
}
}
for (int i = 0; i < processExecutionResult.FileAccessViolationsNotAllowlisted.Count; i++)
{
// Compare individual fields for ReportedFileAccess since it uses reference
// equality for reported process which would not work for serialization/deserialization
AssertEqual(processExecutionResult.FileAccessViolationsNotAllowlisted[i], deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[i]);
}
// Ensure that reported process instances are deduplicated.
XAssert.AreSame(deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[0].Process,
deserializedProcessExecutionResult.FileAccessViolationsNotAllowlisted[2].Process);
for (int i = 0; i < processExecutionResult.DynamicallyObservedFiles.Length; i++)
{
AssertEqual(processExecutionResult.DynamicallyObservedFiles[i], deserializedProcessExecutionResult.DynamicallyObservedFiles[i]);
}
for (int i = 0; i < processExecutionResult.DynamicallyProbedFiles.Length; i++)
{
AssertEqual(processExecutionResult.DynamicallyProbedFiles[i], deserializedProcessExecutionResult.DynamicallyProbedFiles[i]);
}
for (int i = 0; i < processExecutionResult.DynamicallyObservedEnumerations.Length; i++)
{
AssertEqual(processExecutionResult.DynamicallyObservedEnumerations[i], deserializedProcessExecutionResult.DynamicallyObservedEnumerations[i]);
}
XAssert.AreSetsEqual(processExecutionResult.AllowedUndeclaredReads, deserializedProcessExecutionResult.AllowedUndeclaredReads, expectedResult: true);
var referencedContentLength = processExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent.Length;
for (int i = 0; i < referencedContentLength; i++)
{
XAssert.AreEqual(
processExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent[i],
deserializedProcessExecutionResult.TwoPhaseCachingInfo.CacheEntry.ReferencedContent[i]);
}
XAssert.AreEqual(9, deserializedProcessExecutionResult.PipProperties["Bar"]);
}
public void PipHistoricPerfDataConstructorDoesntCrash()
{
// TODO: Use IntelliTest/Pex for this.
var times = new[]
{
new DateTime(DateTime.MinValue.Year + 1, 1, 1).ToUniversalTime(),
new DateTime(2015, 1, 1).ToUniversalTime(),
new DateTime(DateTime.MaxValue.Year, 1, 1).ToUniversalTime()
};
var spans = new[] { TimeSpan.Zero, TimeSpan.FromSeconds(1), TimeSpan.MaxValue };
var ints = new[] { 0, int.MaxValue };
// Let's say that the highest possible memory usage is currently 1TB.
var ulongs = new ulong[] { 0, 1, (ulong)1024 * 1024 * 1024 * 1024 };
var uints = new uint[] { 0, 1, uint.MaxValue };
foreach (var executionStart in times)
{
foreach (var executionStop in times)
{
foreach (var processExecutionTime in spans)
{
foreach (var fileMonitoringWarnings in ints)
{
foreach (var ioCounters in new[]
{
new IOCounters(
readCounters: new IOTypeCounters(operationCount: 1, transferCount: ulong.MaxValue),
writeCounters: new IOTypeCounters(operationCount: 0, transferCount: 0),
otherCounters: new IOTypeCounters(operationCount: 0, transferCount: 0)
),
new IOCounters(
readCounters: new IOTypeCounters(operationCount: 0, transferCount: 0),
writeCounters: new IOTypeCounters(operationCount: 0, transferCount: 0),
otherCounters: new IOTypeCounters(operationCount: 0, transferCount: 0)
)
})
{
foreach (var userTime in spans)
{
foreach (var kernelTime in spans)
{
foreach (var peakMemoryUsage in ulongs)
{
foreach (var numberOfProcesses in uints)
{
foreach (var workerId in uints)
{
if (executionStart > executionStop)
{
continue;
}
var performance = new ProcessPipExecutionPerformance(
PipExecutionLevel.Executed,
executionStart,
executionStop,
FingerprintUtilities.ZeroFingerprint,
processExecutionTime,
new FileMonitoringViolationCounters(fileMonitoringWarnings, fileMonitoringWarnings, fileMonitoringWarnings),
ioCounters,
userTime,
kernelTime,
ProcessMemoryCounters.CreateFromBytes(peakMemoryUsage, peakMemoryUsage, peakMemoryUsage),
numberOfProcesses,
workerId);
var data = new PipHistoricPerfData(performance);
data = data.Merge(data);
Analysis.IgnoreResult(data);
}
}
}
}
}
}
}
}
}
}
}
