public void CreateReplacementFileRecreatesWhenDenyWriteACLPresent()
{
const string Target = @"Target";
FileId originalId;
using (FileStream original = File.Create(GetFullPath(Target)))
{
originalId = FileUtilities.ReadFileUsnByHandle(original.SafeFileHandle).Value.FileId;
}
AddDenyWriteACL(GetFullPath(Target));
using (FileStream fs = FileUtilities.CreateReplacementFile(GetFullPath(Target), FileShare.Read | FileShare.Delete))
{
XAssert.IsNotNull(fs);
XAssert.AreNotEqual(originalId, FileUtilities.ReadFileUsnByHandle(fs.SafeFileHandle).Value.FileId, "File was truncated rather than replaced");
XAssert.AreEqual(0, fs.Length);
fs.WriteByte(1);
}
}
public async Task CopyCompletionCallback()
{
const string Src = "src";
const string Target = "target";
File.WriteAllText(GetFullPath(Src), "Source");
Usn closeUsn = default(Usn);
var completionHandlerCalled = false;
await FileUtilities.CopyFileAsync(
GetFullPath(Src),
GetFullPath(Target),
onCompletion : (source, dest) =>
{
completionHandlerCalled = true;
if (!OperatingSystemHelper.IsUnixOS)
{
Usn?maybeCloseUsn = FileUtilities.TryWriteUsnCloseRecordByHandle(dest);
XAssert.IsNotNull(maybeCloseUsn);
closeUsn = maybeCloseUsn.Value;
}
});
XAssert.IsTrue(completionHandlerCalled);
XAssert.IsTrue(File.Exists(GetFullPath(Target)));
XAssert.AreEqual("Source", File.ReadAllText(GetFullPath(Target)));
if (!OperatingSystemHelper.IsUnixOS)
{
XAssert.AreNotEqual(0, closeUsn, "Completion callback skipped");
using (FileStream dest = File.OpenRead(GetFullPath(Target)))
{
Usn usn = FileUtilities.ReadFileUsnByHandle(dest.SafeFileHandle).Value.Usn;
XAssert.AreEqual(closeUsn, usn, "CLOSE usn should have been written during the callback.");
}
}
}
public void MoveTempDeletionCanReplaceRunningExecutable()
{
// Make a copy of DummyWaiter.exe to use as the test subject for deleting a running executable.
// Keep the copy in the same directory as the original since it will need runtime dlls
string dummyWaiterLocation = DummyWaiter.GetDummyWaiterExeLocation();
string exeCopy = dummyWaiterLocation + ".copy.exe";
File.Copy(dummyWaiterLocation, exeCopy);
using (var waiter = DummyWaiter.RunAndWait(exeCopy))
{
BuildXLException caughtException = null;
try
{
FileUtilities.DeleteFile(exeCopy);
}
catch (BuildXLException ex)
{
caughtException = ex;
}
XAssert.IsNotNull(caughtException, "Expected deletion without a tempCleaner to fail");
XAssert.IsTrue(File.Exists(exeCopy));
caughtException = null;
try
{
FileUtilities.DeleteFile(exeCopy, tempDirectoryCleaner: MoveDeleteCleaner);
}
catch (BuildXLException ex)
{
caughtException = ex;
}
XAssert.IsNull(caughtException, "Expected deletion with a MoveDeleteCleaner to succeed");
XAssert.IsFalse(File.Exists(exeCopy));
}
}
public void PreventResize()
{
var list = new ConcurrentArrayList <TestValue>(1, false);
list[0] = new TestValue(0);
XAssert.IsNotNull(list[0]);
XAssert.AreEqual(0, list[0].Value);
Exception expectedException = null;
try
{
list[1] = new TestValue(1);
}
catch (Exception e)
{
expectedException = e;
}
XAssert.IsNotNull(expectedException as ArgumentException);
XAssert.AreEqual("index", ((ArgumentException)expectedException).ParamName);
}
public void TestMultipleReuses()
{
SetupHelloWorld();
SetUpConfig();
EngineState firstEngineState = RunEngine("First build");
FreshSetUp("bar");
EngineState secondEngineState = RunEngine("Second build", engineState: firstEngineState);
XAssert.IsTrue(!EngineState.IsUsable(firstEngineState));
XAssert.IsNotNull(secondEngineState);
XAssert.IsTrue(!secondEngineState.IsDisposed);
XAssert.AreNotSame(firstEngineState, secondEngineState);
FreshSetUp("baz");
EngineState thirdEngineState = RunEngine("Third build", engineState: secondEngineState);
XAssert.IsTrue(!EngineState.IsUsable(secondEngineState));
XAssert.IsNotNull(EngineState.IsUsable(thirdEngineState));
XAssert.IsTrue(!thirdEngineState.IsDisposed);
XAssert.AreNotSame(secondEngineState, thirdEngineState);
}
public async Task TestGcPrefix()
{
string cacheConfig = TestType.NewCache(nameof(TestGcPrefix), true);
BasicFilesystemCache cache = (await InitializeCacheAsync(cacheConfig).SuccessAsync()) as BasicFilesystemCache;
XAssert.IsNotNull(cache, "Failed to create cache for GC tests!");
PipDefinition[] pips =
{
new PipDefinition("Pip1", pipSize: 3),
new PipDefinition("Pip2", pipSize: 4),
new PipDefinition("Pip3", pipSize: 5)
};
// First, lets filter the GC to only do one of the files.
CacheEntries files = await BuildPipsAndGetCacheEntries(cache, "Build", pips);
cache.DeleteSession("Build");
for (int i = 1; i < 4; i++)
{
files.AgeAll();
string targetFile = files.FingerprintFiles.Keys.First();
// Get the shard directory of the weak fingerprint
string prefix = Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(targetFile)));
XAssert.AreEqual(3, prefix.Length);
m_output.WriteLine("GC Prefix: [{0}]", prefix);
var stats = cache.CollectUnreferencedFingerprints(m_output, prefixFilter: prefix.Substring(0, i));
XAssert.IsFalse(File.Exists(targetFile), "Should have moved this one to pending");
BasicFilesystemCache.UndoPendingDelete(targetFile);
files.AssertExists();
}
AssertSuccess(await cache.ShutdownAsync());
}
public void RetryEmptyDirectoryDelete()
{
// Create an empty directory
string dir = Path.Combine(TemporaryDirectory, "dir");
Directory.CreateDirectory(dir);
SafeFileHandle childHandle = null;
FileUtilities.TryCreateOrOpenFile(
dir,
FileDesiredAccess.GenericRead,
FileShare.Read,
FileMode.Open,
FileFlagsAndAttributes.FileFlagBackupSemantics,
out childHandle);
using (childHandle)
{
Exception exception = null;
try
{
// Fails because of handle open to /dir
FileUtilities.DeleteDirectoryContents(dir, deleteRootDirectory: true);
}
catch (Exception e)
{
exception = e;
}
XAssert.IsNotNull(exception);
XAssert.IsTrue(FileUtilities.Exists(dir));
}
AssertVerboseEventLogged(NativeLogEventId.RetryOnFailureException, Helpers.DefaultNumberOfAttempts);
FileUtilities.DeleteDirectoryContents(dir, deleteRootDirectory: true);
XAssert.IsFalse(FileUtilities.Exists(dir));
}
public virtual async Task ReadOnlyRemoteIsNotUpdatedWhenDisconnected()
{
string testCacheId = "Disconnected";
ICache testCache = await InitializeCacheAsync(NewCache(testCacheId, false)).SuccessAsync();
VerticalCacheAggregator vertCache = testCache as VerticalCacheAggregator;
XAssert.IsNotNull(vertCache);
PoisonAllRemoteSessions(testCache);
DisconnectRemoteCache(testCache);
ICacheSession session = (await testCache.CreateSessionAsync()).Success();
FullCacheRecord cacheRecord = await FakeBuild.DoPipAsync(session, "TestPip");
await VerticalAggregatorBaseTests.ValidateItemsInCacheAsync(
vertCache.LocalCache,
cacheRecord.StrongFingerprint.WeakFingerprint,
new List <CasHash>(cacheRecord.CasEntries),
CacheDeterminism.None,
cacheRecord.StrongFingerprint.CasElement,
vertCache.LocalCache.CacheId,
1);
var remoteSession = await vertCache.RemoteCache.CreateReadOnlySessionAsync().SuccessAsync();
int fingerprintsReturned = 0;
foreach (var fingerprint in remoteSession.EnumerateStrongFingerprints(cacheRecord.StrongFingerprint.WeakFingerprint))
{
fingerprintsReturned++;
}
XAssert.AreEqual(0, fingerprintsReturned, "No fingerprints should have been found in the remote cache.");
AssertSuccess(await testCache.ShutdownAsync());
}
public async Task TestCasOnlyShardFile()
{
string baseCacheDir = null;
m_shardFileSource = (cacheDir) =>
{
baseCacheDir = cacheDir;
StringBuilder sb = new StringBuilder();
sb.AppendLine(BasicFilesystemCache.WFP_TOKEN);
sb.AppendLine(BasicFilesystemCache.END_TOKEN);
sb.AppendLine(BasicFilesystemCache.CAS_TOKEN);
sb.AppendLine(cacheDir + "CasDir");
sb.AppendLine(BasicFilesystemCache.END_TOKEN);
return(sb.ToString());
};
string cacheConfig = NewCache("TestCasOnlyShardFile", false);
BasicFilesystemCache cache = (await InitializeCacheAsync(cacheConfig).SuccessAsync()) as BasicFilesystemCache;
XAssert.IsNotNull(cache, "Failed to create cache for tests!");
string casPath = Path.Combine(baseCacheDir + "CasDir", BasicFilesystemCache.CAS_HASH_TOKEN);
foreach (string oneShardDir in cache.CasRoots)
{
XAssert.AreEqual(casPath, oneShardDir, "Cas Directory not as expected");
}
string wfpPath = Path.Combine(baseCacheDir, BasicFilesystemCache.WFP_TOKEN);
foreach (string oneShardDir in cache.FingerprintRoots)
{
XAssert.AreEqual(wfpPath, oneShardDir, "Fingerprint directory not as expected");
}
}
private void WithVolumeHandle(Action <SafeFileHandle> action)
{
VolumeMap map = JournalUtils.TryCreateMapOfAllLocalVolumes(new LoggingContext("Dummy", "Dummy"));
XAssert.IsNotNull(map, "Failed to create a volume map");
using (VolumeAccessor volumeAccessor = map.CreateVolumeAccessor())
{
SafeFileHandle directoryHandle;
var directoryOpenResult = FileUtilities.TryOpenDirectory(
TemporaryDirectory,
FileShare.ReadWrite | FileShare.Delete,
out directoryHandle);
using (directoryHandle)
{
XAssert.IsTrue(directoryOpenResult.Succeeded, "Failed to open the temporary directory to query its volume membership");
SafeFileHandle volumeHandle = volumeAccessor.TryGetVolumeHandle(directoryHandle);
XAssert.IsNotNull(volumeHandle, "Failed to open a volume handle");
action(volumeHandle);
}
}
}
public async Task Deterministic()
{
const string TestName = nameof(Deterministic);
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
ICacheSession session = await cache.CreateSessionAsync().SuccessAsync();
PipDefinition[] pips =
{
new PipDefinition("PipA"),
new PipDefinition("PipB")
};
FullCacheRecord[] records = (await pips.BuildAsync(session)).ToArray();
XAssert.AreEqual(2, records.Length);
// Now, we should be able to redo those adds but swap the CasEntries in the records
// and get them replaced
for (int i = 0; i < 2; i++)
{
// Make sure we have our own strong fingerprint (no cheating by the cache for this one)
var strong = records[i].StrongFingerprint;
strong = new StrongFingerprint(strong.WeakFingerprint, strong.CasElement, strong.HashElement, "testing");
// Validate that the GetCacheEntry produces what we expect
var entries = await session.GetCacheEntryAsync(strong).SuccessAsync();
XAssert.AreEqual(records[i].CasEntries, entries);
// Validate that the other record I am going to do is different
var other = records[1 - i].CasEntries;
XAssert.AreNotEqual(entries, other, "Other entries must be different!");
var newRecord = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, other).SuccessAsync();
XAssert.IsNotNull(newRecord.Record, "Should have returned prior version");
XAssert.AreEqual(records[i], newRecord.Record);
var fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(other, CacheDeterminism.SinglePhaseNonDeterministic));
XAssert.IsFalse(fail.Succeeded, "Should not have succeeded in replacing normal with SinglePhaseNonDeterministic");
XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType(), fail.Failure.Describe());
// Should not matter if the CasEntries are the same or not
fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(entries, CacheDeterminism.SinglePhaseNonDeterministic));
XAssert.IsFalse(fail.Succeeded, "Should not have succeeded in replacing normal with SinglePhaseNonDeterministic");
XAssert.AreEqual(typeof(SinglePhaseMixingFailure), fail.Failure.GetType(), fail.Failure.Describe());
}
// Now for tool deterministic rules
PipDefinition[] toolPips =
{
new PipDefinition("ToolPipA", determinism: CacheDeterminism.Tool),
new PipDefinition("ToolPipB", determinism: CacheDeterminism.Tool)
};
records = (await toolPips.BuildAsync(session)).ToArray();
XAssert.AreEqual(2, records.Length);
// Now, we should not be able to change tool deterministic results as that is
// a major error (not just a failure to add with prior record returned)
for (int i = 0; i < 2; i++)
{
// Make sure we have our own strong fingerprint (no cheating by the cache for this one)
var strong = records[i].StrongFingerprint;
strong = new StrongFingerprint(strong.WeakFingerprint, strong.CasElement, strong.HashElement, "testing");
// Validate that the GetCacheEntry produces what we expect
var entries = await session.GetCacheEntryAsync(strong).SuccessAsync();
XAssert.AreEqual(records[i].CasEntries, entries);
// Validate that the other record I am going to do is different
var other = records[1 - i].CasEntries;
XAssert.AreNotEqual(entries, other, "Other entries must be different!");
// Setting our own should be just fine (same content with tool determinism)
var newRecord = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, entries).SuccessAsync();
XAssert.IsNull(newRecord.Record);
// A non-tool deteriminism update should have returned the prior version
newRecord = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, new CasEntries(other, CacheDeterminism.None)).SuccessAsync();
XAssert.AreEqual(records[i], newRecord.Record);
// Giving a tool deterministic different answer should fail
var fail = await session.AddOrGetAsync(strong.WeakFingerprint, strong.CasElement, strong.HashElement, other);
XAssert.IsFalse(fail.Succeeded, "Should have failed when trying to give conflicting tool deterministic entries");
XAssert.AreEqual(typeof(NotDeterministicFailure), fail.Failure.GetType(), fail.Failure.Describe());
}
await session.CloseAsync().SuccessAsync();
await ShutdownCacheAsync(cache, testCacheId);
}
public async Task CheckProcessTreeTimoutOnNestedChildProcessTimeoutWhenRootProcessExitedAsync()
{
var processInfo = CreateProcessInfoWithSandboxConnection(Operation.Echo("hi"));
processInfo.NestedProcessTerminationTimeout = TimeSpan.FromMilliseconds(10);
// Set the last enqueue time to now
s_connection.MinReportQueueEnqueueTime = Sandbox.GetMachAbsoluteTime();
using (var process = CreateAndStartSandboxedProcess(processInfo))
{
var time = s_connection.MinReportQueueEnqueueTime;
var childProcessPath = "/dummy/exe2";
var childProcessPid = process.ProcessId + 1;
// first post some reports indicating that
// - a child process was spawned
// - the main process exited
// (not posting that the child process exited)
var postTask1 = GetContinuouslyPostAccessReportsTask(process, new List <ReportInstruction>
{
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpProcessStart,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromMilliseconds(100).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromMilliseconds(200).Ticks * 100),
},
Pid = childProcessPid,
Path = childProcessPath,
Allowed = true
},
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpProcessExit,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromMilliseconds(300).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromMilliseconds(400).Ticks * 100),
},
Pid = process.ProcessId,
Path = "/dummy/exe",
Allowed = true
},
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpKAuthCreateDir,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromMilliseconds(500).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromMilliseconds(600).Ticks * 100),
},
Pid = childProcessPid,
Path = childProcessPath,
Allowed = true
},
});
// SandboxedProcessMac should decide to kill the process because its child survived;
// when it does that, it will call this callback. When that happens, we must post
// OpProcessTreeCompleted because SandboxedProcessMac will keep waiting for it.
s_connection.ProcessTerminated += (pipId, pid) =>
{
postTask1.GetAwaiter().GetResult();
ContinuouslyPostAccessReports(process, new List <ReportInstruction>
{
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpProcessTreeCompleted,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromMilliseconds(900).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromMilliseconds(1000).Ticks * 100),
},
Pid = process.ProcessId,
Path = "/dummy/exe",
Allowed = true
}
});
};
var result = await process.GetResultAsync();
await postTask1; // await here as well just to make AsyncFixer happy
XAssert.IsTrue(result.Killed, "Expected process to have been killed");
XAssert.IsFalse(result.TimedOut, "Didn't expect process to have timed out");
XAssert.IsNotNull(result.SurvivingChildProcesses, "Expected surviving child processes");
XAssert.IsTrue(result.SurvivingChildProcesses.Any(p => p.Path == childProcessPath),
$"Expected surviving child processes to contain {childProcessPath}; " +
$"instead it contains: {string.Join(", ", result.SurvivingChildProcesses.Select(p => p.Path))}");
}
}
public async Task DeterminismNotUpgraded(int fromDeterminism, int toDeterminism, bool differentCasEntries)
{
string testName = I($"DeterminismNotUpgraded{fromDeterminism}x{toDeterminism}{(differentCasEntries ? "Diff" : "Same")}");
string testCacheId = MakeCacheId(testName);
ICache cache = await CreateCacheAsync(testCacheId);
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
// We need at least 2 to make "differentCasEntries" work
PipDefinition[] pips =
{
new PipDefinition("PipA", determinism: s_determinism[fromDeterminism]),
new PipDefinition("PipB", determinism: s_determinism[fromDeterminism])
};
var records = (await pips.BuildAsync(session)).ToArray();
await CloseSessionAsync(session, testSessionId);
testSessionId = "Session2-" + testCacheId;
session = await CreateSessionAsync(cache, testSessionId);
// What we will do here is AddOrGet() a record with the determinism bit changed.
for (int i = 0; i < records.Length; i++)
{
var record = records[i];
// This gets the CasEntries we want
CasEntries newEntries = records[(i + (differentCasEntries ? 1 : 0)) % records.Length].CasEntries;
// Validate that the entry for the record is what we expect
var entries = (await session.GetCacheEntryAsync(record.StrongFingerprint)).Success();
XAssert.AreEqual(s_determinism[fromDeterminism].EffectiveGuid, entries.Determinism.EffectiveGuid);
// Now pin the CasElement and all of the CasEntries
(await session.PinToCasAsync(record.StrongFingerprint.CasElement, CancellationToken.None)).Success();
(await session.PinToCasAsync(newEntries, CancellationToken.None)).Success();
// Now make a new record
var newRecord = (await session.AddOrGetAsync(
record.StrongFingerprint.WeakFingerprint,
record.StrongFingerprint.CasElement,
record.StrongFingerprint.HashElement,
new CasEntries(newEntries, s_determinism[toDeterminism]))).Success();
// The new record should be null since the contents were the same.
if (differentCasEntries)
{
XAssert.IsNotNull(newRecord.Record);
XAssert.AreEqual(record, newRecord.Record);
}
else
{
XAssert.IsNull(newRecord.Record);
}
// Now, we will try to get the same record from the cache to validate
// the setting of the bit
entries = (await session.GetCacheEntryAsync(record.StrongFingerprint)).Success();
XAssert.AreEqual(record.CasEntries, entries);
XAssert.AreEqual(s_determinism[fromDeterminism].EffectiveGuid, entries.Determinism.EffectiveGuid);
}
await CloseSessionAsync(session, testSessionId);
await ShutdownCacheAsync(cache, testCacheId);
}
public async Task DisconnectMostRemoteAfterBuildReturnsMostRemoteCacheDeterminism()
{
string cacheId = "MutlipleCacheRemote";
ICache testCache = await InitializeCacheAsync(NewCache(cacheId, true, false, true)).SuccessAsync();
VerticalCacheAggregator lowerVert = testCache as VerticalCacheAggregator;
XAssert.IsNotNull(lowerVert);
CallbackCacheWrapper callbackCache = lowerVert.RemoteCache as CallbackCacheWrapper;
XAssert.IsNotNull(callbackCache);
VerticalCacheAggregator upperVert = callbackCache.WrappedCache as VerticalCacheAggregator;
XAssert.IsNotNull(upperVert);
ICacheSession session = await testCache.CreateSessionAsync().SuccessAsync();
FullCacheRecord cacheRecord = await FakeBuild.DoPipAsync(session, "Test Pip");
VerticalAggregatorDisconnectTests.DisconnectCache(upperVert.RemoteCache);
// Now query each cache, and verify only the remote content is in each.
var remoteDeterminism = CacheDeterminism.ViaCache(upperVert.RemoteCache.CacheGuid, CacheDeterminism.NeverExpires);
foreach (var currentCache in new Tuple <ICache, CacheDeterminism, string, int>[]
{
new Tuple <ICache, CacheDeterminism, string, int>(testCache, remoteDeterminism, lowerVert.LocalCache.CacheId, 2),
new Tuple <ICache, CacheDeterminism, string, int>(lowerVert.LocalCache, remoteDeterminism, lowerVert.LocalCache.CacheId, 1),
new Tuple <ICache, CacheDeterminism, string, int>(upperVert.LocalCache, remoteDeterminism, upperVert.LocalCache.CacheId, 1),
new Tuple <ICache, CacheDeterminism, string, int>(upperVert.RemoteCache, remoteDeterminism, upperVert.RemoteCache.CacheId, 1),
})
{
await ValidateItemsInCacheAsync(
currentCache.Item1,
cacheRecord.StrongFingerprint.WeakFingerprint,
new List <CasHash>(cacheRecord.CasEntries),
currentCache.Item2,
cacheRecord.StrongFingerprint.CasElement,
currentCache.Item3,
currentCache.Item4);
}
// And make sure it flips back on re-connect.
remoteDeterminism = CacheDeterminism.ViaCache(upperVert.RemoteCache.CacheGuid, CacheDeterminism.NeverExpires);
VerticalAggregatorDisconnectTests.ConnectCache(upperVert.RemoteCache);
foreach (var currentCache in new Tuple <ICache, CacheDeterminism, string, int>[]
{
new Tuple <ICache, CacheDeterminism, string, int>(testCache, CacheDeterminism.ViaCache(upperVert.RemoteCache.CacheGuid, CacheDeterminism.NeverExpires), lowerVert.LocalCache.CacheId, 3),
new Tuple <ICache, CacheDeterminism, string, int>(lowerVert.LocalCache, CacheDeterminism.ViaCache(upperVert.RemoteCache.CacheGuid, CacheDeterminism.NeverExpires), lowerVert.LocalCache.CacheId, 1),
new Tuple <ICache, CacheDeterminism, string, int>(upperVert.LocalCache, CacheDeterminism.ViaCache(upperVert.RemoteCache.CacheGuid, CacheDeterminism.NeverExpires), upperVert.LocalCache.CacheId, 1),
new Tuple <ICache, CacheDeterminism, string, int>(upperVert.RemoteCache, remoteDeterminism, upperVert.RemoteCache.CacheId, 1),
})
{
await ValidateItemsInCacheAsync(
currentCache.Item1,
cacheRecord.StrongFingerprint.WeakFingerprint,
new List <CasHash>(cacheRecord.CasEntries),
currentCache.Item2,
cacheRecord.StrongFingerprint.CasElement,
currentCache.Item3,
currentCache.Item4);
}
await testCache.ShutdownAsync().SuccessAsync();
}
public async Task TestGcMulti()
{
// To keep test time down, we do all of the "safely skipped during GC" tests in one go
// This is actually safe as I then validate that it recovers later. Under normal conditions
// this rarely happens but we want to make sure that the GC can handle the concurrent access
// characteristics and this hits those other code paths.
string cacheConfig = TestType.NewCache(nameof(TestGcMulti), true);
BasicFilesystemCache cache = (await InitializeCacheAsync(cacheConfig).SuccessAsync()) as BasicFilesystemCache;
XAssert.IsNotNull(cache, "Failed to create cache for GC tests!");
foreach (var build in DoBuilds(cache, 4).OutOfOrderTasks())
{
await build;
}
// Now delete the sessions one at a time and age at each step
for (int i = 0; i < 4; i++)
{
new CacheEntries(cache).AgeAll();
FullGc(cache);
cache.DeleteSession("Build" + i);
}
// At this point we should have deleted a few items at each level
// The last session should still have been alive at the last GC
// so we should now have both pending and non-pending in both the
// fingerprints and CAS
CacheEntries files = new CacheEntries(cache);
XAssert.IsTrue(files.FingerprintFiles.Keys.Any(IsPendingDelete));
XAssert.IsTrue(files.FingerprintFiles.Keys.Any(IsNotPendingDelete));
XAssert.IsTrue(files.CasFiles.Keys.Any(IsPendingDelete));
XAssert.IsTrue(files.CasFiles.Keys.Any(IsNotPendingDelete));
// Age them all
files.AgeAll();
// Now, we need to open some files of each time and run a GC to see that it works.
using (var notPendingFingerprint = files.FingerprintFiles.First(kv => IsNotPendingDelete(kv.Key)).Value.OpenRead())
{
using (var pendingFingerprint = files.FingerprintFiles.First(kv => IsPendingDelete(kv.Key)).Value.OpenRead())
{
using (var notPendingCas = files.CasFiles.First(kv => IsNotPendingDelete(kv.Key)).Value.OpenRead())
{
using (var pendingCas = files.CasFiles.First(kv => IsPendingDelete(kv.Key)).Value.OpenRead())
{
// This should allow the GC to continue but some things will not be collected
// to do failed deletes and failed renames to pending
var stats = FullGc(cache);
XAssert.AreEqual(1, stats["CAS_Skipped"], "Should have skipped a CAS entry");
XAssert.AreEqual(2, stats["Fingerprint_Skipped"], "Should have skipped 2 Fingerprints");
}
}
}
}
// Now with the files not blocked from delete/rename, run the GC again (no fresh aging)
var statsRemaining = FullGc(cache);
XAssert.AreEqual(1, statsRemaining["Fingerprint_Collected"], "Collected the one fingerprint that was held open");
XAssert.AreEqual(1, statsRemaining["Fingerprint_Pending"], "Moved to pending the one fingerprint that was held open");
XAssert.AreEqual(1, statsRemaining["CAS_Collected"], "Collected the one CAS item that was held open");
XAssert.AreEqual(4, statsRemaining["CAS_Pending"], "Moved to pending the one CAS item that was held open and the 3 CAS items that were referenced by the fingerprint that was held");
// Check that the partition counts match as needed
XAssert.AreEqual((TestType is TestBasicFilesystemSharded) ? TestBasicFilesystemSharded.SHARD_COUNT : 1, statsRemaining["Fingerprint_Partitions"]);
XAssert.AreEqual((TestType is TestBasicFilesystemSharded) ? TestBasicFilesystemSharded.SHARD_COUNT : 1, statsRemaining["CAS_Partitions"]);
AssertSuccess(await cache.ShutdownAsync());
}
public async Task TestGcBasic()
{
string cacheConfig = TestType.NewCache(nameof(TestGcBasic), true);
BasicFilesystemCache cache = (await InitializeCacheAsync(cacheConfig).SuccessAsync()) as BasicFilesystemCache;
XAssert.IsNotNull(cache, "Failed to create cache for GC tests!");
// Verify that we don't have prior content in the cache
XAssert.AreEqual(0, new CacheEntries(cache).Count, "Test cache did not start out empty!");
PipDefinition[] pipsSession1 =
{
new PipDefinition("Pip1", pipSize: 3),
new PipDefinition("Pip2", pipSize: 4),
new PipDefinition("Pip3", pipSize: 5)
};
CacheEntries session1Files = await BuildPipsAndGetCacheEntries(cache, "Session1", pipsSession1);
// Nothing should change on this GC since the files and are all referenced
FullGc(cache);
session1Files.AssertExists();
session1Files.AgeAll();
// Everything is rooted so nothing should happen here
FullGc(cache);
XAssert.IsFalse(new CacheEntries(cache).AreDifferences(session1Files), "We changed the cache when we should not have!");
// Now, if we delete the session, we should collect things.
cache.DeleteSession("Session1");
FullGc(cache);
// All of the fingerprints should be changed to pending but nothing should be deleted
CacheEntries session1gcFpPending = new CacheEntries(cache);
XAssert.AreEqual(session1Files.Count, session1gcFpPending.Count, "Nothing should have been added or deleted!");
XAssert.IsFalse(session1gcFpPending.FingerprintFiles.Keys.Any(IsNotPendingDelete), "All fingerprints should be pending delete");
XAssert.IsFalse(session1gcFpPending.CasFiles.Keys.Any(IsPendingDelete), "All cas should not be pending delete");
// Nothing to happen here as the pending files are too new
FullGc(cache);
// Nothing changed...
XAssert.IsFalse(new CacheEntries(cache).AreDifferences(session1gcFpPending), "We changed the cache when we should not have!");
// Now age the pending delete such that they are collected
session1gcFpPending.AgeAll();
FullGc(cache);
CacheEntries session1gcCas1 = new CacheEntries(cache);
XAssert.AreEqual(0, session1gcCas1.FingerprintFiles.Count, "Should have collected all fingerprints");
// And, we should have moved to pending all CAS items (since there is no pending)
XAssert.IsFalse(session1gcCas1.CasFiles.Keys.Any(IsNotPendingDelete), "All cas should be pending delete");
FullGc(cache); // Should do nothing as they are not old enough pending
XAssert.IsFalse(new CacheEntries(cache).AreDifferences(session1gcCas1), "We changed the cache when we should not have!");
// After getting all to be old, this should finally GC it all
session1gcCas1.AgeAll();
FullGc(cache);
XAssert.AreEqual(0, new CacheEntries(cache).Count, "Should have collected everything.");
AssertSuccess(await cache.ShutdownAsync());
}
public async Task TestGcConcurrency()
{
// This is just minimal concurrency testing such that it does not completely mess up
// Most of the work in making the GC safe was in the design and hand testing as this
// is about having a GC run while multiple mutators run (and multiple GCs run)
string cacheConfig = TestType.NewCache(nameof(TestGcConcurrency), true);
BasicFilesystemCache cache = (await InitializeCacheAsync(cacheConfig).SuccessAsync()) as BasicFilesystemCache;
XAssert.IsNotNull(cache, "Failed to create cache for GC tests!");
PipDefinition[] pips =
{
new PipDefinition("Pip1", pipSize: 3),
new PipDefinition("Pip2", pipSize: 4),
new PipDefinition("Pip3", pipSize: 5),
new PipDefinition("Pip4", pipSize: 4),
new PipDefinition("Pip5", pipSize: 3)
};
CacheEntries files = await BuildPipsAndGetCacheEntries(cache, "Build", pips);
// First have the GC do nothing (all items still rooted but old enough to collect)
files.AgeAll();
ParallelConcurrentFullGc(cache, "Noop");
files.AssertExists();
// Now delete the session and make sure we collect correctly
cache.DeleteSession("Build");
// This may take a few tries to get all of the items GC'ed
// Since we assume that the GC will always work, any error returned
// by the GC will trigger a fault and failure of the test.
// However, the GC is designed to, if anything is in question at
// all, to just skip deleting (or marking as pending delete)
// any item that is busy or otherwise engaged. This does mean that
// multiple concurrent GC can, in rare cases, cause a file
// (specifically a strong fingerprint) to not get collected for
// a given pass due to the file being in use by another GC.
// This is perfectly correct and will cause that item to be
// collected later. However, later may require another
// aging of the files since the file age may have been
// reset during the failed attempt to mark it pending.
// (Which is a good thing since failing to mark pending is
// a potential sign that something is using it and thus
// it is not yet ready to be changed)
// Anyway, the number of passes and the exact state of the GC
// is not directly knowable due to these races but they sure should
// not be greater than 100
int pass = 0;
while (files.Count > 0)
{
files.AgeAll();
pass++;
ParallelConcurrentFullGc(cache, "Pass #" + pass);
// Make sure some progress happened
// That means some items had to be marked pending or
// got collected each time through the process.
// This will make sure we are always making some
// progress with the GC and thus will terminate.
files.AssertMissingSome();
files = new CacheEntries(cache);
}
AssertSuccess(await cache.ShutdownAsync());
}
private static void AssertEqual(HistoricTableSizes expected, HistoricTableSizes actual)
{
XAssert.IsNotNull(expected);
XAssert.IsNotNull(actual);
XAssert.ArrayEqual(expected.ToArray(), actual.ToArray());
}
private void AssertAlmostEqualToDefaultConfigurationObject(IConfiguration conf, params string[] exceptions)
{
XAssert.IsNotNull(conf);
AssertAlmostEqual(conf, m_defaultConf, exceptions);
}
/// <summary>
/// Verify that graph is successfully constructed.
/// </summary>
private void VerifyGraphSuccessfullyConstructed(PipGraph graph)
{
XAssert.IsNotNull(graph, "Failed in constructing graph");
}
public void SerializeSandboxedProcessInfo(bool useNullFileStorage, bool useRootJail)
{
var pt = new PathTable();
var fam =
new FileAccessManifest(pt, CreateDirectoryTranslator())
{
FailUnexpectedFileAccesses = false,
IgnoreCodeCoverage = false,
ReportFileAccesses = false,
ReportUnexpectedFileAccesses = false,
MonitorChildProcesses = false
};
var vac = new ValidationDataCreator(fam, pt);
vac.AddScope(A("C", "Users", "AppData"), FileAccessPolicy.AllowAll);
vac.AddPath(A("C", "Source", "source.txt"), FileAccessPolicy.AllowReadAlways);
vac.AddPath(A("C", "Out", "out.txt"), FileAccessPolicy.AllowAll);
SandboxedProcessStandardFiles standardFiles = null;
ISandboxedProcessFileStorage fileStorage;
if (useNullFileStorage)
{
fileStorage = null;
}
else
{
standardFiles = new SandboxedProcessStandardFiles(A("C", "pip", "pip.out"), A("C", "pip", "pip.err"));
fileStorage = new StandardFileStorage(standardFiles);
}
var envVars = new Dictionary <string, string>()
{
["Var1"] = "Val1",
["Var2"] = "Val2",
};
IBuildParameters buildParameters = BuildParameters.GetFactory().PopulateFromDictionary(envVars);
var sidebandLogFile = A("C", "engine-cache", "sideband-logs", "log-1");
var loggerRootDirs = new[] { A("C", "out", "dir1"), A("C", "out", "dir2") };
var sharedOpaqueOutputLogger = new SidebandWriter(DefaultSidebandMetadata, sidebandLogFile, loggerRootDirs);
SandboxedProcessInfo info = new SandboxedProcessInfo(
pt,
fileStorage,
A("C", "tool", "tool.exe"),
fam,
true,
null,
LoggingContext,
sidebandWriter: sharedOpaqueOutputLogger)
{
Arguments = @"/arg1:val1 /arg2:val2",
WorkingDirectory = A("C", "Source"),
RootJailInfo = useRootJail ? (RootJailInfo?)new RootJailInfo(A("C", "RootJail"), 123, 234) : null,
EnvironmentVariables = buildParameters,
Timeout = TimeSpan.FromMinutes(15),
PipSemiStableHash = 0x12345678,
PipDescription = nameof(SerializeSandboxedProcessInfo),
TimeoutDumpDirectory = A("C", "Timeout"),
SandboxKind = global::BuildXL.Utilities.Configuration.SandboxKind.Default,
AllowedSurvivingChildProcessNames = new[] { "conhost.exe", "mspdbsrv.exe" },
NestedProcessTerminationTimeout = SandboxedProcessInfo.DefaultNestedProcessTerminationTimeout,
StandardInputSourceInfo = StandardInputInfo.CreateForData("Data"),
StandardObserverDescriptor = new SandboxObserverDescriptor()
{
WarningRegex = new ExpandedRegexDescriptor("*warn", System.Text.RegularExpressions.RegexOptions.Compiled)
},
};
// Serialize and deserialize.
SandboxedProcessInfo readInfo = null;
using (var stream = new MemoryStream())
{
info.Serialize(stream);
stream.Position = 0;
readInfo = SandboxedProcessInfo.Deserialize(
stream,
new global::BuildXL.Utilities.Instrumentation.Common.LoggingContext("Test"),
null);
}
using (readInfo.SidebandWriter)
{
// Verify.
XAssert.AreEqual(info.FileName, readInfo.FileName);
XAssert.AreEqual(info.Arguments, readInfo.Arguments);
XAssert.AreEqual(info.WorkingDirectory, readInfo.WorkingDirectory);
XAssert.AreEqual(info.RootJailInfo?.RootJail, readInfo.RootJailInfo?.RootJail);
XAssert.AreEqual(info.RootJailInfo?.UserId, readInfo.RootJailInfo?.UserId);
XAssert.AreEqual(info.RootJailInfo?.GroupId, readInfo.RootJailInfo?.GroupId);
var readEnvVars = readInfo.EnvironmentVariables.ToDictionary();
XAssert.AreEqual(envVars.Count, readEnvVars.Count);
foreach (var kvp in envVars)
{
XAssert.AreEqual(kvp.Value, readEnvVars[kvp.Key]);
}
XAssert.AreEqual(info.Timeout, readInfo.Timeout);
XAssert.AreEqual(info.PipSemiStableHash, readInfo.PipSemiStableHash);
XAssert.AreEqual(info.PipDescription, readInfo.PipDescription);
XAssert.AreEqual(info.TimeoutDumpDirectory, readInfo.TimeoutDumpDirectory);
XAssert.AreEqual(info.SandboxKind, readInfo.SandboxKind);
XAssert.AreEqual(info.AllowedSurvivingChildProcessNames.Length, readInfo.AllowedSurvivingChildProcessNames.Length);
for (int i = 0; i < info.AllowedSurvivingChildProcessNames.Length; ++i)
{
XAssert.AreEqual(info.AllowedSurvivingChildProcessNames[i], readInfo.AllowedSurvivingChildProcessNames[i]);
}
XAssert.AreEqual(info.NestedProcessTerminationTimeout, readInfo.NestedProcessTerminationTimeout);
XAssert.AreEqual(info.StandardInputSourceInfo, readInfo.StandardInputSourceInfo);
if (useNullFileStorage)
{
XAssert.IsNull(readInfo.SandboxedProcessStandardFiles);
XAssert.IsNull(readInfo.FileStorage);
}
else
{
XAssert.IsNotNull(readInfo.SandboxedProcessStandardFiles);
XAssert.AreEqual(standardFiles.StandardOutput, readInfo.SandboxedProcessStandardFiles.StandardOutput);
XAssert.AreEqual(standardFiles.StandardError, readInfo.SandboxedProcessStandardFiles.StandardError);
XAssert.AreEqual(standardFiles.StandardOutput, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardOutput));
XAssert.AreEqual(standardFiles.StandardError, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardError));
}
XAssert.IsFalse(readInfo.ContainerConfiguration.IsIsolationEnabled);
XAssert.AreEqual(sidebandLogFile, readInfo.SidebandWriter.SidebandLogFile);
XAssert.ArrayEqual(loggerRootDirs, readInfo.SidebandWriter.RootDirectories.ToArray());
if (!OperatingSystemHelper.IsUnixOS)
{
// this validator examines serialized FAM bytes using the same Windows-only native parser used by Detours
ValidationDataCreator.TestManifestRetrieval(vac.DataItems, readInfo.FileAccessManifest, false);
}
}
}
private async Task VerifyReporting(AccessType access, Action <FileAccessManifest> populateManifest, params ExpectedReportEntry[] expectedExplicitReports)
{
// We need a process which will generate the expected accesses.
var testProcess = CreateTestProcess(access, expectedExplicitReports);
var pathTable = Context.PathTable;
var info = ToProcessInfo(testProcess, "FileAccessExplicitReportingTest");
info.FileAccessManifest.ReportFileAccesses = false;
info.FileAccessManifest.ReportUnexpectedFileAccesses = true;
info.FileAccessManifest.FailUnexpectedFileAccesses = false;
populateManifest(info.FileAccessManifest);
using (ISandboxedProcess process = await StartProcessAsync(info))
{
SandboxedProcessResult result = await process.GetResultAsync();
XAssert.AreEqual(0, result.ExitCode,
"\r\ncmd: {0} \r\nStandard out: '{1}' \r\nStandard err: '{2}'.",
info.Arguments,
await result.StandardOutput.ReadValueAsync(),
await result.StandardError.ReadValueAsync());
XAssert.IsNotNull(result.ExplicitlyReportedFileAccesses);
Dictionary <AbsolutePath, ExpectedReportEntry> pathsToExpectations = expectedExplicitReports.ToDictionary(
e => e.File.Path,
e => e);
var verifiedPaths = new HashSet <AbsolutePath>();
foreach (var actualReport in result.ExplicitlyReportedFileAccesses)
{
string actualReportedPathString = actualReport.GetPath(pathTable);
XAssert.AreEqual(
FileAccessStatus.Allowed,
actualReport.Status,
"Incorrect status for path " + actualReportedPathString);
if (!TryVerifySingleReport(pathTable, actualReport, access, pathsToExpectations, out var actualReportPath))
{
if ((actualReport.RequestedAccess & RequestedAccess.Enumerate) != 0)
{
// To account for 'explicitly reported' enumerations globally, we need to be lenient about unexpected enumerations.
// Alternatively instead opt-in to enumeration reports under certain scopes.
}
else
{
AbsolutePath actualReportedPath = AbsolutePath.Create(pathTable, actualReportedPathString);
XAssert.Fail("No expectations for an explicitly reported path {0}", actualReportedPath.ToString(pathTable));
}
}
else
{
verifiedPaths.Add(actualReportPath);
}
}
foreach (var actualReport in result.AllUnexpectedFileAccesses)
{
XAssert.AreEqual(FileAccessStatus.Denied, actualReport.Status);
if (TryVerifySingleReport(pathTable, actualReport, access, pathsToExpectations, out var actualReportPath))
{
verifiedPaths.Add(actualReportPath);
}
// Note that we allow extra unexpected file accesses for the purposes of these tests.
}
var expectedPathsSet = new HashSet <AbsolutePath>(pathsToExpectations.Keys);
var disagreeingPaths = new HashSet <AbsolutePath>(verifiedPaths);
disagreeingPaths.SymmetricExceptWith(expectedPathsSet);
if (disagreeingPaths.Any())
{
var disagreeingReports = "Disagreeing reports:" + string.Join(string.Empty, disagreeingPaths
.Select(p => (tag: expectedPathsSet.Contains(p) ? "Missing" : "Unexpected", path: p))
.Select(t => $"{Environment.NewLine} {t.tag} report for path {t.path.ToString(pathTable)}"));
var expectedReports = "Expected reports:" + string.Join(string.Empty, pathsToExpectations.Keys
.Select(p => $"{Environment.NewLine} {p.ToString(pathTable)}"));
var verifiedReports = "Verified reports:" + string.Join(string.Empty, verifiedPaths
.Select(p => $"{Environment.NewLine} {p.ToString(pathTable)}"));
XAssert.Fail(string.Join(Environment.NewLine, disagreeingReports, expectedReports, verifiedReports));
}
}
}
protected override Task CorruptCasEntry(ICache cache, CasHash hash)
{
XAssert.IsNotNull(cache as MemCache, "Invalid cache passed to TestInMemory CorruptCasEntry test method");
return(CorruptEntry(cache, hash));
}
public async Task TestGcMultiSession()
{
string cacheConfig = TestType.NewCache(nameof(TestGcMultiSession), true);
BasicFilesystemCache cache = (await InitializeCacheAsync(cacheConfig).SuccessAsync()) as BasicFilesystemCache;
XAssert.IsNotNull(cache, "Failed to create cache for GC tests!");
// Verify that we don't have prior content in the cache
XAssert.AreEqual(0, new CacheEntries(cache).Count, "Test cache did not start out empty!");
// Also used for session 3
PipDefinition[] pipsSession1 =
{
new PipDefinition("Pip1", pipSize: 3),
new PipDefinition("Pip2", pipSize: 4),
new PipDefinition("Pip3", pipSize: 5)
};
// This should just bring back from the "pending" one pip with 4 outputs
// Also used for session 4
PipDefinition[] pipsSession2 =
{
new PipDefinition("Pip2", pipSize: 4)
};
CacheEntries session1Files = await BuildPipsAndGetCacheEntries(cache, "Session1", pipsSession1);
CacheEntries session2Files = await BuildPipsAndGetCacheEntries(cache, "Session2", pipsSession2);
// The second session should not have changed anything as the pip already existed
XAssert.IsFalse(session2Files.AreDifferences(session1Files), "We changed the cache when we should not have!");
// Nothing should change on this GC since the files are all referenced
session1Files.AgeAll();
FullGc(cache);
session1Files.AssertExists();
// Nothing should change because of session 2 deleting since the fingerprint still exists in session1
cache.DeleteSession("Session2");
FullGc(cache);
session1Files.AssertExists();
// Deleteing session 1 should cause all fingerprints to become pending delete
cache.DeleteSession("Session1");
FullGc(cache);
CacheEntries session1GcFpPending = new CacheEntries(cache);
XAssert.IsFalse(session1GcFpPending.FingerprintFiles.Keys.Any(IsNotPendingDelete), "All fingerprints should be pending delete");
XAssert.IsFalse(session1GcFpPending.CasFiles.Keys.Any(IsPendingDelete), "All cas should not be pending delete");
// Rebuilding the pips from session1a should restore the pending delete to non-pending delete (in fact, restore to session1Files)
CacheEntries session3Files = await BuildPipsAndGetCacheEntries(cache, "Session3", pipsSession1);
XAssert.IsFalse(session3Files.AreDifferences(session1Files), "Should be back to the same after rebuilding - no pending");
cache.DeleteSession("Session3");
session3Files.AgeAll();
FullGc(cache);
CacheEntries session3GcFpPending = new CacheEntries(cache);
XAssert.IsFalse(session3GcFpPending.FingerprintFiles.Keys.Any(IsNotPendingDelete), "All fingerprints should be pending delete");
XAssert.IsFalse(session3GcFpPending.CasFiles.Keys.Any(IsPendingDelete), "All cas should not be pending delete");
// Build the session2 single pip (as session 4) to recover the pending
CacheEntries session4Files = await BuildPipsAndGetCacheEntries(cache, "Session4", pipsSession2);
XAssert.AreEqual(session1Files.Count, session4Files.Count, "Should not have made any extra files");
XAssert.AreEqual(1, session4Files.FingerprintFiles.Keys.Count(IsNotPendingDelete), "Should have 1 non-pending delete fingerprint");
// This should collect all but the one fingerprint from session 4 and mark pending all of the cas entries
// except the 5 cas entries from session 4. (4 cas outputs plus the cas input list in the fingerprint)
session4Files.AgeAll();
FullGc(cache);
CacheEntries session4Gc = new CacheEntries(cache);
XAssert.AreEqual(1, session4Gc.FingerprintFiles.Count, "Should only have one fingerprint file left");
XAssert.AreEqual(session1Files.CasFiles.Count, session4Gc.CasFiles.Count);
XAssert.AreEqual(5, session4Gc.CasFiles.Keys.Count(IsNotPendingDelete), "Only Pip2 cas should be non-pending");
cache.DeleteSession("Session4");
// Pip2 fingerprint to pending
session4Gc.AgeAll();
FullGc(cache);
// Pip2 fingerprint from pending to delete - cas entries to pending
new CacheEntries(cache).AgeAll();
FullGc(cache);
CacheEntries session4GcCasPending = new CacheEntries(cache);
XAssert.AreEqual(0, session4GcCasPending.FingerprintFiles.Count, "All fingerprints should be gone");
XAssert.IsFalse(session4GcCasPending.CasFiles.Keys.Any(IsNotPendingDelete), "All cas should be pending delete");
// Pip2 cas entries from pending to delete
session4GcCasPending.AgeAll();
FullGc(cache);
XAssert.AreEqual(0, new CacheEntries(cache).Count, "All should be collected now");
AssertSuccess(await cache.ShutdownAsync());
}
public void ChildProcessCanBreakawayWhenConfigured(bool letInfiniteWaiterSurvive)
{
// Skip this test if running on .NET Framework with vstest
// Reason: when this is the case and code coverage is turned on, launching breakaway
// processes here causes the code coverage monitoring process to hang.
if (!OperatingSystemHelper.IsDotNetCore && IsRunningInVsTestTestHost())
{
return;
}
// We use InfiniteWaiter (a process that waits forever) as a long-living process that we can actually check it can
// escape the job object
var fam = new FileAccessManifest(
Context.PathTable,
childProcessesToBreakawayFromSandbox: letInfiniteWaiterSurvive ? new[] { InfiniteWaiterToolName } : null)
{
FailUnexpectedFileAccesses = false
};
// We instruct the regular test process to spawn InfiniteWaiter as a child
var info = ToProcessInfo(
ToProcess(
Operation.SpawnExe(
Context.PathTable,
CreateFileArtifactWithName(InfiniteWaiterToolName, TestDeploymentDir))),
fileAccessManifest: fam);
// Let's shorten the default time to wait for nested processes, since we are spawning
// a process that never ends and we don't want this test to wait for that long
info.NestedProcessTerminationTimeout = TimeSpan.FromMilliseconds(10);
var result = RunProcess(info).GetAwaiter().GetResult();
if (result.ExitCode != 0)
{
XAssert.Fail(
$"Process exited with exit code {result.ExitCode}." +
$"\n\n=== stdout ===\n\n ${result.StandardOutput.ReadValueAsync().Result}" +
$"\n\n=== stderr ===\n\n ${result.StandardError.ReadValueAsync().Result}");
}
if (!letInfiniteWaiterSurvive)
{
// If we didn't let infinite waiter escape, we should have killed it when the job object was finalized
XAssert.IsTrue(result.Killed);
XAssert.IsNotNull(result.SurvivingChildProcesses);
XAssert.Contains(
result.SurvivingChildProcesses.Select(p => p?.Path).Where(p => p != null).Select(p => System.IO.Path.GetFileName(p).ToUpperInvariant()),
InfiniteWaiterToolName.ToUpperInvariant());
}
else
{
// If we did let it escape, then nothing should have been killed (nor tried to survive and later killed, from the job object point of view)
XAssert.IsFalse(result.Killed);
if (result.SurvivingChildProcesses != null && result.SurvivingChildProcesses.Any())
{
var survivors = string.Join(
", ",
result.SurvivingChildProcesses.Select(p => p?.Path != null ? System.IO.Path.GetFileName(p.Path) : "<unknown>"));
XAssert.Fail($"Unexpected {result.SurvivingChildProcesses.Count()} surviving child processes: {survivors}");
}
// Let's retrieve the child process and confirm it survived
var infiniteWaiterInfo = RetrieveChildProcessesCreatedBySpawnExe(result).Single();
// The fact that this does not throw confirms survival
var dummyWaiter = Process.GetProcessById(infiniteWaiterInfo.pid);
try
{
// Just being protective, let's make sure we are talking about the same process
XAssert.AreEqual(infiniteWaiterInfo.processName, dummyWaiter.ProcessName);
}
finally
{
// Now let's kill the surviving process, since we don't want it to linger around unnecessarily
dummyWaiter.Kill();
}
}
}
public async Task CheckProcessTreeTimoutOnNestedChildProcessTimeoutWhenRootProcessExitedAsync()
{
var processInfo = CreateProcessInfoWithSandboxConnection(Operation.Echo("hi"));
processInfo.NestedProcessTerminationTimeout = TimeSpan.FromMilliseconds(100);
// Set the last enqueue time to now
s_connection.MinReportQueueEnqueueTime = Sandbox.GetMachAbsoluteTime();
var process = CreateAndStartSandboxedProcess(processInfo);
var taskCancelationSource = new CancellationTokenSource();
var time = Sandbox.GetMachAbsoluteTime();
var childProcessPath = "/dummy/exe2";
var instructions = new List <ReportInstruction>()
{
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpProcessStart,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromSeconds(1).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromSeconds(2).Ticks * 100),
},
Pid = 1235,
Path = childProcessPath,
Allowed = true
},
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpProcessExit,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromSeconds(3).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromSeconds(4).Ticks * 100),
},
Pid = process.ProcessId,
Path = "/dummy/exe",
Allowed = true
},
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpKAuthCreateDir,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromSeconds(5).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromSeconds(6).Ticks * 100),
},
Pid = 1235,
Path = childProcessPath,
Allowed = true
},
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpProcessExit,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromSeconds(7).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromSeconds(8).Ticks * 100),
},
Pid = 1235,
Path = childProcessPath,
Allowed = true
},
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpProcessTreeCompleted,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromSeconds(9).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromSeconds(10).Ticks * 100),
},
Pid = process.ProcessId,
Path = "/dummy/exe",
Allowed = true
},
};
ContinouslyPostAccessReports(process, taskCancelationSource.Token, instructions);
var result = await process.GetResultAsync();
taskCancelationSource.Cancel();
XAssert.IsTrue(result.Killed, "Expected process to have been killed");
XAssert.IsFalse(result.TimedOut, "Didn't expect process to have timed out");
XAssert.IsNotNull(result.SurvivingChildProcesses, "Expected surviving child processes");
XAssert.IsTrue(result.SurvivingChildProcesses.Any(p => p.Path == childProcessPath),
$"Expected surviving child processes to contain {childProcessPath}; " +
$"instead it contains: {string.Join(", ", result.SurvivingChildProcesses.Select(p => p.Path))}");
}
public void TestEngineStateFileContentTableReuse()
{
SetupHelloWorld();
SetUpConfig();
EngineState lastEngineState = RunEngine("First build");
var firstFCT = lastEngineState.FileContentTable;
XAssert.IsNotNull(firstFCT);
FileIdAndVolumeId inFileIdentity = GetIdentity(GetFullPath(InputFilename));
FileIdAndVolumeId outFileIdentity = GetIdentity(GetFullPath(OutputFilename));
Usn inFileUsn = Usn.Zero;
Usn outFileUsn = Usn.Zero;
ISet <FileIdAndVolumeId> ids = new HashSet <FileIdAndVolumeId>();
XAssert.IsTrue(FileContentTableAccessorFactory.TryCreate(out var accesor, out string error));
firstFCT.VisitKnownFiles(accesor, FileShare.ReadWrite | FileShare.Delete,
(fileIdAndVolumeId, fileHandle, path, knownUsn, knownHash) =>
{
if (fileIdAndVolumeId == inFileIdentity)
{
inFileUsn = knownUsn;
}
else if (fileIdAndVolumeId == outFileIdentity)
{
outFileUsn = knownUsn;
}
ids.Add(fileIdAndVolumeId);
return(true);
});
XAssert.AreNotEqual(Usn.Zero, inFileUsn);
XAssert.AreNotEqual(Usn.Zero, outFileUsn);
// Run engine again
FreshSetUp("change some stuff");
lastEngineState = RunEngine("Second build", engineState: lastEngineState);
var secondFCT = lastEngineState.FileContentTable;
XAssert.AreNotSame(firstFCT, secondFCT); // The FCT gets updated at the end of the run
outFileIdentity = GetIdentity(GetFullPath(OutputFilename)); // Output file changed
bool visitedInput = false;
bool visitedOutput = false;
secondFCT.VisitKnownFiles(accesor, FileShare.ReadWrite | FileShare.Delete,
(fileIdAndVolumeId, fileHandle, path, knownUsn, knownHash) =>
{
if (fileIdAndVolumeId == inFileIdentity)
{
XAssert.IsTrue(ids.Contains(fileIdAndVolumeId));
XAssert.IsTrue(inFileUsn < knownUsn); // We modified the file
inFileUsn = knownUsn;
visitedInput = true;
}
else if (fileIdAndVolumeId == outFileIdentity)
{
XAssert.IsFalse(ids.Contains(fileIdAndVolumeId));
XAssert.IsTrue(outFileUsn < knownUsn); // New output file
outFileUsn = knownUsn;
visitedOutput = true;
}
else
{
XAssert.IsTrue(ids.Contains(fileIdAndVolumeId)); // Other entries are still there
}
return(true);
});
XAssert.IsTrue(visitedInput);
XAssert.IsTrue(visitedOutput);
XAssert.IsTrue(inFileUsn < outFileUsn);
XAssert.AreEqual(firstFCT.Count + 1, secondFCT.Count); // There's a new entry because the new output file has a different fileId
}
public async Task LoadOrCreateHandlesNonExistentTable()
{
FileContentTable table = await LoadOrCreateTable();
XAssert.IsNotNull(table);
}
public void SerializeSandboxedProcessInfo()
{
var pt = new PathTable();
var fam =
new FileAccessManifest(pt, CreateDirectoryTranslator())
{
FailUnexpectedFileAccesses = false,
IgnoreCodeCoverage = false,
ReportFileAccesses = false,
ReportUnexpectedFileAccesses = false,
MonitorChildProcesses = false
};
var vac = new ValidationDataCreator(fam, pt);
vac.AddScope(A("C", "Users", "AppData"), FileAccessPolicy.AllowAll);
vac.AddPath(A("C", "Source", "source.txt"), FileAccessPolicy.AllowReadAlways);
vac.AddPath(A("C", "Out", "out.txt"), FileAccessPolicy.AllowAll);
var standardFiles = new SandboxedProcessStandardFiles(A("C", "pip", "pip.out"), A("C", "pip", "pip.err"));
var envVars = new Dictionary <string, string>()
{
["Var1"] = "Val1",
["Var2"] = "Val2",
};
IBuildParameters buildParameters = BuildParameters.GetFactory().PopulateFromDictionary(envVars);
SandboxedProcessInfo info = new SandboxedProcessInfo(
pt,
new StandardFileStorage(standardFiles),
A("C", "tool", "tool.exe"),
fam,
true,
null)
{
Arguments = @"/arg1:val1 /arg2:val2",
WorkingDirectory = A("C", "Source"),
EnvironmentVariables = buildParameters,
Timeout = TimeSpan.FromMinutes(15),
PipSemiStableHash = 0x12345678,
PipDescription = nameof(SerializeSandboxedProcessInfo),
ProcessIdListener = null,
TimeoutDumpDirectory = A("C", "Timeout"),
SandboxKind = global::BuildXL.Utilities.Configuration.SandboxKind.Default,
AllowedSurvivingChildProcessNames = new[] { "conhost.exe", "mspdbsrv.exe" },
NestedProcessTerminationTimeout = SandboxedProcessInfo.DefaultNestedProcessTerminationTimeout,
StandardInputSourceInfo = StandardInputInfo.CreateForData("Data"),
StandardObserverDescriptor = new SandboxObserverDescriptor()
{
WarningRegex = new ExpandedRegexDescriptor("*warn", System.Text.RegularExpressions.RegexOptions.Compiled)
},
};
// Serialize and deserialize.
SandboxedProcessInfo readInfo = null;
using (var stream = new MemoryStream())
{
info.Serialize(stream);
stream.Position = 0;
readInfo = SandboxedProcessInfo.Deserialize(
stream,
new global::BuildXL.Utilities.Instrumentation.Common.LoggingContext("Test"),
null);
}
// Verify.
XAssert.AreEqual(info.FileName, readInfo.FileName);
XAssert.AreEqual(info.Arguments, readInfo.Arguments);
XAssert.AreEqual(info.WorkingDirectory, readInfo.WorkingDirectory);
var readEnvVars = readInfo.EnvironmentVariables.ToDictionary();
XAssert.AreEqual(envVars.Count, readEnvVars.Count);
foreach (var kvp in envVars)
{
XAssert.AreEqual(kvp.Value, readEnvVars[kvp.Key]);
}
XAssert.AreEqual(info.Timeout, readInfo.Timeout);
XAssert.AreEqual(info.PipSemiStableHash, readInfo.PipSemiStableHash);
XAssert.AreEqual(info.PipDescription, readInfo.PipDescription);
XAssert.AreEqual(info.ProcessIdListener, readInfo.ProcessIdListener);
XAssert.AreEqual(info.TimeoutDumpDirectory, readInfo.TimeoutDumpDirectory);
XAssert.AreEqual(info.SandboxKind, readInfo.SandboxKind);
XAssert.AreEqual(info.AllowedSurvivingChildProcessNames.Length, readInfo.AllowedSurvivingChildProcessNames.Length);
for (int i = 0; i < info.AllowedSurvivingChildProcessNames.Length; ++i)
{
XAssert.AreEqual(info.AllowedSurvivingChildProcessNames[i], readInfo.AllowedSurvivingChildProcessNames[i]);
}
XAssert.AreEqual(info.NestedProcessTerminationTimeout, readInfo.NestedProcessTerminationTimeout);
XAssert.AreEqual(info.StandardInputSourceInfo, readInfo.StandardInputSourceInfo);
XAssert.IsNotNull(readInfo.SandboxedProcessStandardFiles);
XAssert.AreEqual(standardFiles.StandardOutput, readInfo.SandboxedProcessStandardFiles.StandardOutput);
XAssert.AreEqual(standardFiles.StandardError, readInfo.SandboxedProcessStandardFiles.StandardError);
XAssert.AreEqual(standardFiles.StandardOutput, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardOutput));
XAssert.AreEqual(standardFiles.StandardError, readInfo.FileStorage.GetFileName(SandboxedProcessFile.StandardError));
XAssert.IsFalse(readInfo.ContainerConfiguration.IsIsolationEnabled);
ValidationDataCreator.TestManifestRetrieval(vac.DataItems, readInfo.FileAccessManifest, false);
}
