public async Task SimpleDummySession()
{
const string TestName = "SimpleSession";
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
// Now for the session (which we base on the cache ID)
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
// Do the default fake build for this test (first time, no cache hit)
FullCacheRecord built = await FakeBuild.DoPipAsync(session, TestName);
XAssert.AreEqual(FakeBuild.NewRecordCacheId, built.CacheId, "Should have been a new cache entry!");
// Now we see if we can get back the items we think we should
await CloseSessionAsync(session, testSessionId);
// We need a read only session to get the CasEntries
ICacheReadOnlySession readOnlySession = (await cache.CreateReadOnlySessionAsync()).Success();
// Validate that the cache contains a dummy session and it has the one cache record it needs.
HashSet <FullCacheRecord> found = new HashSet <FullCacheRecord>();
foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(MemoizationStoreAdapterCache.DummySessionName).Success().OutOfOrderTasks())
{
StrongFingerprint strongFingerprint = await strongFingerprintTask;
CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success();
FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries);
// If it is not the record we already found...
if (!found.Contains(record))
{
found.Add(record);
}
XAssert.AreEqual(1, found.Count, "There should be only 1 unique record in the session");
XAssert.AreEqual(built.StrongFingerprint.WeakFingerprint, record.StrongFingerprint.WeakFingerprint);
XAssert.AreEqual(built.StrongFingerprint.CasElement, record.StrongFingerprint.CasElement);
XAssert.AreEqual(built.StrongFingerprint.HashElement, record.StrongFingerprint.HashElement);
XAssert.AreEqual(built.CasEntries.Count, record.CasEntries.Count, "Did not return the same number of items");
XAssert.IsTrue(record.CasEntries.Equals(built.CasEntries), "Items returned are not the same hash and/or order order");
XAssert.AreEqual(built, record);
// We can not check record.CasEntries.IsDeterministic
// as the cache may have determined that they are deterministic
// via cache determinism recovery.
}
XAssert.AreEqual(1, found.Count, "There should be 1 and only 1 record in the session!");
await readOnlySession.CloseAsync().SuccessAsync();
// Check that the cache has the items in it
await FakeBuild.CheckContentsAsync(cache, built);
// Now redo the "build" with a cache hit
testSessionId = "Session2-" + testCacheId;
session = await CreateSessionAsync(cache, testSessionId);
FullCacheRecord rebuilt = await FakeBuild.DoPipAsync(session, TestName);
XAssert.AreEqual(built, rebuilt, "Should have been the same build!");
// We make sure we did get it from a cache rather than a manual rebuild.
XAssert.AreNotEqual(built.CacheId, rebuilt.CacheId, "Should not be the same cache ID");
await CloseSessionAsync(session, testSessionId);
readOnlySession = await cache.CreateReadOnlySessionAsync().SuccessAsync();
// Now that we have done the second build via a cache hit, it should produce the
// same cache record as before
foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(MemoizationStoreAdapterCache.DummySessionName).Success().OutOfOrderTasks())
{
StrongFingerprint strongFingerprint = await strongFingerprintTask;
CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success();
FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries);
XAssert.IsTrue(found.Contains(record), "Second session should produce the same cache record but did not!");
}
(await readOnlySession.CloseAsync()).Success();
await ShutdownCacheAsync(cache, testCacheId);
}
public void TestAddingDirectoryContainingFilesWithAbsentFileHash(bool isSourceFile)
{
string remoteDirectoryPath = "remoteDirectory";
string fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
var dropPaths = new List <string>();
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.Associated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
var file = new SealedDirectoryFile(
Path.Combine(directoryPath, "file.txt"),
new FileArtifact(new AbsolutePath(1), isSourceFile ? 0 : 1),
FileContentInfo.CreateWithUnknownLength(WellKnownContentHashes.AbsentFile));
return(IpcResult.Success(cmd.RenderResult(new List <SealedDirectoryFile> {
file
})));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = CreateDummyBxlApiClient(ipcProvider, moniker);
WithSetup(
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter .*",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(dropPaths.Count == 0);
// if an absent file is a source file, drop operation should have failed; otherwise, we simply skip it
XAssert.AreEqual(!isSourceFile, ipcResult.Succeeded);
XAssert.AreEqual(isSourceFile ? IpcResultStatus.InvalidInput : IpcResultStatus.Success, ipcResult.ExitCode);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
}
public void GetOrSetStress()
{
// Generate a bunch of operations that work on a set of fields that trigger resizes, updates and reads.
var list = new ConcurrentArrayList <TestValue>(1, true);
var rnd = new Random();
int maxIndex = 2000;
int maxOperationsFactor = 1000;
int val0 = 0xfff0000;
int val1 = 0x000ff000;
int val2 = 0x00000fff;
var operations = Enumerable
.Range(0, maxIndex * maxOperationsFactor)
.Select(
index =>
{
int value;
int valueToSet = rnd.Next(3);
switch (valueToSet)
{
case 0:
value = val0;
break;
case 1:
value = val1;
break;
case 2:
value = val2;
break;
default:
value = 0;
XAssert.Fail("sdfsd");
break;
}
return(new
{
Index = index / maxOperationsFactor,
Value = value
});
})
.ToArray()
.OrderBy(x => rnd.Next(maxIndex));
Parallel.ForEach(
operations,
operation =>
{
// Make sure it is one of the legal value.
TestValue res = list.GetOrSet(operation.Index, () => new TestValue(operation.Value));
XAssert.IsTrue(res.Value == val0 || res.Value == val1 || res.Value == val2);
// Also inspect random location. Make sure it is a legal value (or null when it is not set yet)
TestValue other = list[rnd.Next(maxIndex)];
XAssert.IsTrue(other == null || other.Value == val0 || other.Value == val1 || other.Value == val2);
});
}
public void ReportedFileAccessCreate()
{
var pathTable = new PathTable();
AbsolutePath file1 = AbsolutePath.Create(pathTable, A("t", "file1.txt"));
var process = new ReportedProcess(0, string.Empty);
ReportedFileAccess rfa1 = ReportedFileAccess.Create(
ReportedFileOperation.CreateFile,
process,
RequestedAccess.Read,
FileAccessStatus.Allowed,
true,
0,
ReportedFileAccess.NoUsn,
DesiredAccess.GENERIC_READ,
ShareMode.FILE_SHARE_NONE,
CreationDisposition.OPEN_ALWAYS,
FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL,
file1);
XAssert.AreEqual(rfa1.Status, FileAccessStatus.Allowed);
XAssert.AreEqual(rfa1.ManifestPath, file1);
XAssert.AreEqual(rfa1.Path, null);
XAssert.AreEqual(A("t", "file1.txt"), rfa1.GetPath(pathTable));
XAssert.AreEqual(rfa1.OpenedFileOrDirectoryAttributes, (FlagsAndAttributes)FlagsAndAttributesConstants.InvalidFileAttributes);
XAssert.IsFalse(rfa1.IsOpenedHandleDirectory());
ReportedFileAccess rfa2 = ReportedFileAccess.Create(
ReportedFileOperation.CreateFile,
process,
RequestedAccess.Read,
FileAccessStatus.CannotDeterminePolicy,
true,
0,
new Usn(0),
DesiredAccess.GENERIC_READ,
ShareMode.FILE_SHARE_NONE,
CreationDisposition.OPEN_ALWAYS,
FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL,
FlagsAndAttributes.FILE_ATTRIBUTE_DIRECTORY,
pathTable,
A("t", "file1.txt"));
XAssert.AreEqual(rfa2.Status, FileAccessStatus.CannotDeterminePolicy);
XAssert.AreEqual(rfa2.ManifestPath, file1);
XAssert.AreEqual(rfa2.Path, null);
XAssert.AreEqual(A("t", "file1.txt"), rfa2.GetPath(pathTable));
XAssert.AreEqual(rfa2.OpenedFileOrDirectoryAttributes, FlagsAndAttributes.FILE_ATTRIBUTE_DIRECTORY);
XAssert.IsTrue(rfa2.IsOpenedHandleDirectory());
ReportedFileAccess rfa3 = ReportedFileAccess.Create(
ReportedFileOperation.CreateFile,
process,
RequestedAccess.Read,
FileAccessStatus.Denied,
true,
0,
ReportedFileAccess.NoUsn,
DesiredAccess.GENERIC_READ,
ShareMode.FILE_SHARE_NONE,
CreationDisposition.OPEN_ALWAYS,
FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL,
FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL,
pathTable,
A("t", "file2.txt"));
XAssert.AreEqual(rfa3.Status, FileAccessStatus.Denied);
XAssert.AreEqual(rfa3.ManifestPath, AbsolutePath.Invalid);
XAssert.AreEqual(rfa3.Path, A("t", "file2.txt"));
XAssert.AreEqual(A("t", "file2.txt"), rfa3.GetPath(pathTable));
XAssert.AreEqual(rfa3.OpenedFileOrDirectoryAttributes, FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL);
XAssert.IsFalse(rfa3.IsOpenedHandleDirectory());
}
public async Task CorrelateMoveOrRenameDirectoryAsync(bool move)
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
AbsolutePath workDirectory = tempFiles.GetDirectory(pathTable, "Directory");
AbsolutePath sourceDirectory = tempFiles.GetDirectory(pathTable, workDirectory, "SourceDirectory");
AbsolutePath sourceFile = sourceDirectory.Combine(pathTable, "SourceFile.txt");
WriteFile(pathTable, sourceFile, "content");
AbsolutePath destinationDirectory = workDirectory.Combine(pathTable, "DestinationDirectory");
AbsolutePath destinationFile = destinationDirectory.Combine(pathTable, "SourceFile.txt");
var process = CreateDetourProcess(
context,
pathTable,
tempFiles,
argumentStr: move ? "CorrelateMoveDirectory" : "CorrelateRenameDirectory",
inputFiles: ReadOnlyArray <FileArtifact> .Empty,
inputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
outputFiles: ReadOnlyArray <FileArtifactWithAttributes> .Empty,
outputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(workDirectory));
var correlator = new Correlator(pathTable);
SandboxedProcessPipExecutionResult result = await RunProcessAsync(
pathTable : pathTable,
ignoreSetFileInformationByHandle : false,
ignoreZwRenameFileInformation : false,
monitorNtCreate : true,
ignoreReparsePoints : false,
ignoreNonCreateFileReparsePoints : false,
monitorZwCreateOpenQueryFile : false,
context : context,
pip : process,
detoursListener : correlator,
errorString : out _);
VerifyNormalSuccess(context, result);
XAssert.IsTrue(Directory.Exists(destinationDirectory.ToString(pathTable)));
var toVerify = new List <(AbsolutePath, RequestedAccess, FileAccessStatus)>
{
(sourceFile, RequestedAccess.Write, FileAccessStatus.Allowed),
(destinationFile, RequestedAccess.Write, FileAccessStatus.Allowed),
(sourceDirectory, RequestedAccess.Write, FileAccessStatus.Allowed),
(destinationDirectory, RequestedAccess.Write, FileAccessStatus.Allowed)
};
VerifyFileAccesses(context, result.AllReportedFileAccesses, toVerify.ToArray());
correlator.VerifyCorrelation(
new Correlator.VerifiedCorrelation(
destinationFile.ToString(pathTable),
move ? ReportedFileOperation.MoveFileWithProgressDest : ReportedFileOperation.SetFileInformationByHandleDest,
sourceFile.ToString(pathTable),
move ? ReportedFileOperation.MoveFileWithProgressSource : ReportedFileOperation.SetFileInformationByHandleSource),
new Correlator.VerifiedCorrelation(
destinationDirectory.ToString(pathTable),
move ? ReportedFileOperation.MoveFileWithProgressDest : ReportedFileOperation.SetFileInformationByHandleDest,
sourceDirectory.ToString(pathTable),
move ? ReportedFileOperation.MoveFileWithProgressSource : ReportedFileOperation.SetFileInformationByHandleSource));
}
}
public async Task ReadFileRandomAccess()
{
const int NumberOfReads = 16;
const int NumberOfWordsPerRead = 64 * 1024;
const int NumberOfBytesPerRead = NumberOfWordsPerRead * 4;
const int NumberOfWords = NumberOfWordsPerRead * NumberOfReads;
const int TotalSize = NumberOfWords * 4;
string path = GetFullPath("file");
using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Delete))
{
using (var writer = new BinaryWriter(fs, Encoding.UTF8, leaveOpen: true))
{
for (int i = 0; i < NumberOfWords; i++)
{
writer.Write((int)i);
}
}
}
#if !FEATURE_CORECLR
using (var io = new IOCompletionManager())
#else
IIOCompletionManager io = null;
#endif
{
using (
IAsyncFile file = AsyncFileFactory.CreateOrOpen(
path,
FileDesiredAccess.GenericRead,
FileShare.Read | FileShare.Delete,
FileMode.Open,
FileFlagsAndAttributes.None,
io))
{
XAssert.IsTrue(file.CanRead);
XAssert.IsFalse(file.CanWrite);
var readBuffer = new byte[TotalSize];
var readTasks = new Task[NumberOfReads];
for (int i = 0; i < readTasks.Length; i++)
{
int offset = NumberOfBytesPerRead * i;
readTasks[i] = Task.Run(
async() =>
{
byte[] localBuffer = new byte[NumberOfBytesPerRead];
int readSoFar = 0;
while (readSoFar < NumberOfBytesPerRead)
{
FileAsyncIOResult result =
await file.ReadAsync(localBuffer, bytesToRead: NumberOfBytesPerRead - readSoFar, fileOffset: offset + readSoFar);
XAssert.AreEqual(FileAsyncIOStatus.Succeeded, result.Status);
XAssert.IsTrue(result.BytesTransferred > 0);
XAssert.IsTrue(readSoFar + result.BytesTransferred <= NumberOfBytesPerRead);
Buffer.BlockCopy(localBuffer, 0, readBuffer, offset + readSoFar, result.BytesTransferred);
readSoFar += result.BytesTransferred;
}
Contract.Assert(readSoFar == NumberOfBytesPerRead);
});
}
for (int i = 0; i < readTasks.Length; i++)
{
await readTasks[i];
}
using (var reader = new BinaryReader(new MemoryStream(readBuffer, writable: false), Encoding.UTF8, leaveOpen: false))
{
for (int i = 0; i < NumberOfWords; i++)
{
XAssert.AreEqual(i, reader.ReadInt32());
}
}
}
}
}
public async Task Stress()
{
const int N = 5;
const int M = N * N;
var context = BuildXLContext.CreateInstanceForTesting();
var loggingContext = CreateLoggingContextForTest();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var config = ConfigHelpers.CreateDefault(pathTable, tempFiles.GetUniqueFileName(), tempFiles);
using (var pipTable = new PipTable(
context.PathTable,
context.SymbolTable,
initialBufferSize: 1024,
maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3,
debug: false))
{
var executionEnvironment = new PipQueueTestExecutionEnvironment(context, config, pipTable, Path.Combine(TestOutputDirectory, "temp"), GetSandboxConnection());
Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) =>
{
PipResult result;
var operationTracker = new OperationTracker(runnablePip.LoggingContext);
var pip = runnablePip.Pip;
using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext))
{
result = await TestPipExecutor.ExecuteAsync(operationContext, executionEnvironment, pip);
}
executionEnvironment.MarkExecuted(pip);
return(result);
};
string executable = CmdHelper.OsShellExe;
FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
// This is the only file artifact we reference without a producer. Rather than scheduling a hashing pip, let's just invent one (so fingerprinting can succeed).
executionEnvironment.AddWellKnownFile(executableArtifact, WellKnownContentHashes.UntrackedFile);
using (var phase1PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 1: create some files
var baseFileArtifacts = new List <FileArtifact>();
for (int i = 0; i < N; i++)
{
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
baseFileArtifacts.Add(destinationArtifact);
PipData contents = PipDataBuilder.CreatePipData(
context.StringTable,
" ",
PipDataFragmentEscaping.CRuntimeArgumentRules,
i.ToString(CultureInfo.InvariantCulture));
var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
var pipId = pipTable.Add((uint)(i + 1), writeFile);
var contentHash = ContentHashingUtilities.HashString(contents.ToString(pathTable));
executionEnvironment.AddExpectedWrite(writeFile, destinationArtifact, contentHash);
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactory, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(DispatcherKind.IO);
phase1PipQueue.Enqueue(runnable);
}
phase1PipQueue.SetAsFinalized();
phase1PipQueue.DrainQueues();
await Task.WhenAll(
Enumerable.Range(0, 2).Select(
async range =>
{
using (var phase2PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 2: do some more with those files
var pips = new ConcurrentDictionary <PipId, Tuple <string, int> >();
var checkerTasks = new ConcurrentQueue <Task>();
Action <PipId, Task <PipResult> > callback =
(id, task) =>
{
XAssert.IsTrue(task.Status == TaskStatus.RanToCompletion);
XAssert.IsFalse(task.Result.Status.IndicatesFailure());
Tuple <string, int> t;
if (!pips.TryRemove(id, out t))
{
XAssert.Fail();
}
checkerTasks.Enqueue(
Task.Run(
() =>
{
string actual = File.ReadAllText(t.Item1).Trim();
// TODO: Make this async
XAssert.AreEqual(actual, t.Item2.ToString());
}));
};
var r = new Random(0);
for (int i = 0; i < M; i++)
{
int sourceIndex = r.Next(baseFileArtifacts.Count);
FileArtifact sourceArtifact = baseFileArtifacts[sourceIndex];
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
Pip pip;
DispatcherKind queueKind;
switch (r.Next(2))
{
case 0:
pip = new CopyFile(sourceArtifact, destinationArtifact, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
queueKind = DispatcherKind.IO;
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
case 1:
string workingDirectory =
OperatingSystemHelper.IsUnixOS ? "/tmp" :
Environment.GetFolderPath(Environment.SpecialFolder.Windows);
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
var pipData = OperatingSystemHelper.IsUnixOS ?
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-c", "'", "cp", sourceArtifact, destinationArtifact, "'") :
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "/d", "/c", "copy", "/B", sourceArtifact, destinationArtifact);
queueKind = DispatcherKind.CPU;
pip = new Process(
executableArtifact,
workingDirectoryAbsolutePath,
pipData,
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .Empty,
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact, sourceArtifact),
ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(destinationArtifact.WithAttributes()),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)),
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
default:
Contract.Assert(false);
continue;
}
var pipId = pipTable.Add((uint)((range *M) + N + i + 1), pip);
Func <RunnablePip, Task> taskFactoryWithCallback = async(runnablePip) =>
{
var task = taskFactory(runnablePip);
var pipResult = await task;
callback(pipId, task);
};
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactoryWithCallback, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(queueKind);
phase2PipQueue.Enqueue(runnable);
if (!pips.TryAdd(pipId, Tuple.Create(destination, sourceIndex)))
{
Contract.Assert(false);
}
}
phase2PipQueue.SetAsFinalized();
phase2PipQueue.DrainQueues();
XAssert.AreEqual(0, pips.Count);
await Task.WhenAll(checkerTasks);
}
}));
}
}
}
}
public void AllBasicEvents()
{
using (var l = new TestListener())
{
MemberInfo[] allEventsMembers = typeof(Events).GetMembers(BindingFlags.Public | BindingFlags.Instance);
foreach (MemberInfo eventsMember in allEventsMembers)
{
if (eventsMember is MethodInfo)
{
// Not all members are methods.
continue;
}
var eventAttribute = eventsMember.GetCustomAttribute(typeof(EventAttribute)) as EventAttribute;
if (eventAttribute == null)
{
// Not all members are tagged with [Event] data.
continue;
}
var eventMethod = (MethodInfo)eventsMember;
bool containsRelatedActivityId;
object[] payload = CreateEventPayload(eventMethod, out containsRelatedActivityId);
eventMethod.Invoke(Events.Log, payload);
XAssert.AreNotEqual(l.LastEvent.EventId, EventId.None, "Last EventId should not be None. Check payload and Message format.");
XAssert.AreEqual(eventAttribute.EventId, l.LastEvent.EventId, "Wrong event ID logged for event {0}", eventMethod.Name);
XAssert.AreNotEqual(0, eventAttribute.Task, "Invalid task id == 0 for event {0} (this results in auto-assignment of task id)", eventMethod.Name);
XAssert.AreEqual(eventAttribute.Task, l.LastEvent.Task, "Wrong task logged for event {0}", eventMethod.Name);
XAssert.AreEqual(eventAttribute.Opcode, l.LastEvent.Opcode, "Wrong opcode logged for event {0}", eventMethod.Name);
XAssert.AreEqual(eventAttribute.Level, l.LastEvent.Level, "Wrong level logged for event {0}", eventMethod.Name);
XAssert.IsTrue(!containsRelatedActivityId || eventAttribute.Opcode == EventOpcode.Start, "Found related activity ID for non-start event {0}", eventMethod.Name);
// EventSource does something weird with the top bits, but the bottom bits should be non-zero.
unchecked
{
XAssert.AreEqual <uint>(
(uint)eventAttribute.Keywords,
(uint)l.LastEvent.Keywords,
"Wrong keywords logged for event {0}",
eventMethod.Name);
XAssert.AreNotEqual <uint>(0, (uint)eventAttribute.Keywords, "Event {0} should have keywords defined", eventMethod.Name);
}
XAssert.IsTrue(
(eventAttribute.Keywords & Keywords.Diagnostics) == 0 || (eventAttribute.Keywords & Keywords.UserMessage) == 0,
"Event {0} specifies both the UserMessage and Diagnostics keywords; these are mutually exclusive.", eventMethod.Name);
string formattedMessage;
try
{
formattedMessage = string.Format(CultureInfo.InvariantCulture, l.LastEvent.Message, l.LastEvent.Payload.ToArray());
}
catch (FormatException ex)
{
XAssert.Fail(
"Formatting the message for event {0}. Maybe the format string does not match the WriteEvent arguments. Exception: {1}",
eventMethod.Name,
ex.ToString());
return;
}
if (containsRelatedActivityId)
{
object[] payloadCopy = new object[payload.Length - 1];
Array.Copy(payload, 1, payloadCopy, 0, payloadCopy.Length);
payload = payloadCopy;
}
XAssert.AreEqual(
payload.Length,
l.LastEvent.Payload.Count,
"Wrong payload size for event {0} (formatted message: `{1}`)",
eventMethod.Name,
formattedMessage);
for (int i = 0; i < payload.Length; i++)
{
XAssert.AreEqual(
payload[i],
l.LastEvent.Payload[i],
"Incorrect payload value for event {0}, position {1} (formatted message: `{2}`)",
eventMethod.Name,
i,
formattedMessage);
}
// We now verify that all strings in the payload appear in the message.
// We exclude pip-provenance and phase events for now because for some reason they define GUIDs that never actually
// get logged (which is probably not a valid thing to do on an EventSource).
if (!l.LastEvent.Message.StartsWith(EventConstants.ProvenancePrefix, StringComparison.Ordinal) &&
!l.LastEvent.Message.StartsWith(EventConstants.PhasePrefix, StringComparison.Ordinal) &&
!l.LastEvent.Keywords.HasFlag(Keywords.Performance))
{
for (int i = 0; i < payload.Length; i++)
{
var currentPayload = payload[i] as string;
if (currentPayload != null)
{
XAssert.IsTrue(
formattedMessage.Contains(currentPayload),
"The formatted message for event {0} did not include the payload string at index {1}",
eventMethod.Name,
i);
}
}
}
if (eventAttribute.EventId != (int)EventId.ErrorEvent)
{
XAssert.IsTrue(
eventAttribute.Level != EventLevel.Error ||
(eventAttribute.Keywords & Keywords.UserMessage) == Keywords.UserMessage,
"Event {0} marked as Error must be Keywords.UserMessage",
eventMethod.Name);
}
}
}
}
public void TestUnifyIpcPips()
{
// TODO: Add unit test that unify Ipc pips.
XAssert.IsTrue(true);
}
public void TestConcurrentBigMapOperations()
{
var map = new ConcurrentBigMap <int, string>();
XAssert.IsTrue(map.TryAdd(0, "value"));
XAssert.IsFalse(map.TryAdd(0, "not added value"));
XAssert.AreEqual("value", map[0]);
map[0] = "newValue";
map[1] = "value1";
var value0 = "newValue";
XAssert.AreEqual(value0, map[0]);
XAssert.IsTrue(map.ContainsKey(0));
XAssert.IsTrue(map.ContainsKey(1));
XAssert.IsFalse(map.ContainsKey(12));
XAssert.AreEqual(2, map.Count);
// Test TryGetValue
string value1;
XAssert.IsTrue(map.TryGetValue(1, out value1));
XAssert.AreEqual("value1", value1);
string value31;
XAssert.IsFalse(map.TryGetValue(31, out value31));
// Test update
XAssert.IsFalse(map.TryUpdate(1, "notUpdatedValue1", "notActualValue1"));
XAssert.AreEqual("value1", map[1]);
XAssert.IsTrue(map.TryUpdate(1, "updatedValue1", "value1"));
value1 = map[1];
XAssert.AreEqual("updatedValue1", value1);
// Test remove
int beforeFailedRemoveCount = map.Count;
string value23;
XAssert.IsFalse(map.TryRemove(23, out value23));
XAssert.AreEqual(beforeFailedRemoveCount, map.Count);
map.Add(23, "value23");
XAssert.AreEqual(beforeFailedRemoveCount + 1, map.Count);
XAssert.IsTrue(map.TryRemove(23, out value23));
XAssert.AreEqual("value23", value23);
XAssert.AreEqual(beforeFailedRemoveCount, map.Count);
Assert.Equal(new int[] { 0, 1 }, map.Keys.ToArray());
Assert.Equal(new string[] { value0, value1 }, map.Values.ToArray());
XAssert.AreEqual(2, map.Count);
string addedData = "added data";
string notAddedData = "not added data";
var result = map.GetOrAdd(2, addedData, (key, data0) => data0);
XAssert.IsFalse(result.IsFound);
XAssert.AreEqual(addedData, result.Item.Value);
XAssert.AreEqual(addedData, map[2]);
// Ensure entry is not updated for get or add
result = map.GetOrAdd(2, notAddedData, (key, data0) => data0);
XAssert.IsTrue(result.IsFound);
XAssert.AreEqual(addedData, result.Item.Value);
XAssert.AreEqual(addedData, map[2]);
Func <int, string, string, string> updateFunction =
(key, data0, currentValue) => "updated " + currentValue;
var updatedData = updateFunction(2, notAddedData, addedData);
result = map.AddOrUpdate(2, notAddedData, (key, data0) => data0, updateFunction);
XAssert.IsTrue(result.IsFound);
XAssert.AreEqual(addedData, result.OldItem.Value);
XAssert.AreEqual(updatedData, result.Item.Value);
XAssert.AreEqual(updatedData, map[2]);
result = map.AddOrUpdate(3, addedData, (key, data0) => data0, updateFunction);
XAssert.IsFalse(result.IsFound);
XAssert.AreEqual(addedData, result.Item.Value);
XAssert.AreEqual(addedData, map[3]);
TestOperationsHelper(parallel: false);
}
public void CommandLineEmptyFilter()
{
RootFilter filter = ParseFilter(values: null, commandLineFilter: string.Empty, defaultFilter: NegatedTagFilter, out _, out _);
XAssert.IsTrue(filter.IsEmpty);
}
protected static void AssertDependencyAndDependent(TProject dependency, TProject dependent, SchedulingResult <TProject> result)
{
XAssert.IsTrue(IsDependencyAndDependent(dependency, dependent, result));
}
public async Task CheckProcessTreeTimoutOnNestedChildProcessTimeoutWhenRootProcessExitedAsync()
{
var processInfo = CreateProcessInfoWithSandboxConnection(Operation.Echo("hi"));
processInfo.NestedProcessTerminationTimeout = TimeSpan.FromMilliseconds(10);
// Set the last enqueue time to now
s_connection.MinReportQueueEnqueueTime = Sandbox.GetMachAbsoluteTime();
using (var process = CreateAndStartSandboxedProcess(processInfo))
{
var time = s_connection.MinReportQueueEnqueueTime;
var childProcessPath = "/dummy/exe2";
var childProcessPid = process.ProcessId + 1;
// first post some reports indicating that
// - a child process was spawned
// - the main process exited
// (not posting that the child process exited)
var postTask1 = GetContinuouslyPostAccessReportsTask(process, new List <ReportInstruction>
{
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpProcessStart,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromMilliseconds(100).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromMilliseconds(200).Ticks * 100),
},
Pid = childProcessPid,
Path = childProcessPath,
Allowed = true
},
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpProcessExit,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromMilliseconds(300).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromMilliseconds(400).Ticks * 100),
},
Pid = process.ProcessId,
Path = "/dummy/exe",
Allowed = true
},
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpKAuthCreateDir,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromMilliseconds(500).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromMilliseconds(600).Ticks * 100),
},
Pid = childProcessPid,
Path = childProcessPath,
Allowed = true
},
});
// SandboxedProcessMac should decide to kill the process because its child survived;
// when it does that, it will call this callback. When that happens, we must post
// OpProcessTreeCompleted because SandboxedProcessMac will keep waiting for it.
s_connection.ProcessTerminated += (pipId, pid) =>
{
postTask1.GetAwaiter().GetResult();
ContinuouslyPostAccessReports(process, new List <ReportInstruction>
{
new ReportInstruction()
{
Process = process,
Operation = FileOperation.OpProcessTreeCompleted,
Stats = new Sandbox.AccessReportStatistics()
{
EnqueueTime = time + ((ulong)TimeSpan.FromMilliseconds(900).Ticks * 100),
DequeueTime = time + ((ulong)TimeSpan.FromMilliseconds(1000).Ticks * 100),
},
Pid = process.ProcessId,
Path = "/dummy/exe",
Allowed = true
}
});
};
var result = await process.GetResultAsync();
await postTask1; // await here as well just to make AsyncFixer happy
XAssert.IsTrue(result.Killed, "Expected process to have been killed");
XAssert.IsFalse(result.TimedOut, "Didn't expect process to have timed out");
XAssert.IsNotNull(result.SurvivingChildProcesses, "Expected surviving child processes");
XAssert.IsTrue(result.SurvivingChildProcesses.Any(p => p.Path == childProcessPath),
$"Expected surviving child processes to contain {childProcessPath}; " +
$"instead it contains: {string.Join(", ", result.SurvivingChildProcesses.Select(p => p.Path))}");
}
}
public Task BigStrings()
{
var harness = new StringTableTestHarness();
var st = harness.StringTable;
var s1 = new string('x', StringTable.BytesPerBuffer - 1);
var s2 = new string('y', StringTable.BytesPerBuffer);
var s3 = new string('z', StringTable.BytesPerBuffer + 1);
StringId id1 = harness.AddString(s1);
StringId id2 = harness.AddString(s2);
StringId id3 = harness.AddString(s3);
harness.AddString(s3 + "繙");
XAssert.AreEqual(s1.Length, st.GetLength(id1));
XAssert.AreEqual(s2.Length, st.GetLength(id2));
XAssert.AreEqual(s3.Length, st.GetLength(id3));
var buf = new char[StringTable.BytesPerBuffer + 1];
st.CopyString(id1, buf, 0);
for (int j = 0; j < s1.Length; j++)
{
XAssert.AreEqual(s1[j], buf[j]);
}
st.CopyString(id2, buf, 0);
for (int j = 0; j < s2.Length; j++)
{
XAssert.AreEqual(s2[j], buf[j]);
}
st.CopyString(id3, buf, 0);
for (int j = 0; j < s3.Length; j++)
{
XAssert.AreEqual(s3[j], buf[j]);
}
XAssert.IsTrue(st.CaseInsensitiveEquals(id1, st.AddString(s1)));
XAssert.IsTrue(st.CaseInsensitiveEquals(id2, st.AddString(s2)));
XAssert.IsTrue(st.CaseInsensitiveEquals(id3, st.AddString(s3)));
XAssert.IsTrue(st.Equals(s1, id1));
XAssert.IsTrue(st.Equals(s2, id2));
XAssert.IsTrue(st.Equals(s3, id3));
XAssert.IsFalse(st.CaseInsensitiveEquals(id1, st.AddString(s2)));
XAssert.IsFalse(st.CaseInsensitiveEquals(id1, st.AddString(s3)));
XAssert.IsFalse(st.CaseInsensitiveEquals(id2, st.AddString(s1)));
XAssert.IsFalse(st.CaseInsensitiveEquals(id2, st.AddString(s3)));
XAssert.IsFalse(st.CaseInsensitiveEquals(id3, st.AddString(s1)));
XAssert.IsFalse(st.CaseInsensitiveEquals(id3, st.AddString(s2)));
XAssert.IsFalse(st.Equals(s2, id1));
XAssert.IsFalse(st.Equals(s3, id1));
XAssert.IsFalse(st.Equals(s1, id2));
XAssert.IsFalse(st.Equals(s3, id2));
XAssert.IsFalse(st.Equals(s1, id3));
XAssert.IsFalse(st.Equals(s2, id3));
return(harness.RunCommonTestsAsync());
}
public void StrongFingerprintMissWithAugmentedWeakFingerprintPostCacheLookUp()
{
Configuration.Cache.AugmentWeakFingerprintPathSetThreshold = 2;
var directory = CreateUniqueDirectoryArtifact();
var sourceFile = CreateSourceFile(directory.Path);
var readFileA = CreateSourceFile(directory.Path);
var readFileB = CreateSourceFile(directory.Path);
var readFileC = CreateSourceFile(directory.Path);
var sealedDirectory = CreateAndScheduleSealDirectoryArtifact(
directory.Path,
global::BuildXL.Pips.Operations.SealDirectoryKind.SourceAllDirectories);
var builder = CreatePipBuilder(new[]
{
Operation.ReadFileFromOtherFile(sourceFile, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.AddInputDirectory(sealedDirectory);
var process = SchedulePipBuilder(builder).Process;
// Pip will read file source and file A.
// Two-phase cache will contain mapping
// wp => (#{source, A}, sp1)
File.WriteAllText(ArtifactToString(sourceFile), ArtifactToString(readFileA));
RunScheduler().AssertCacheMiss(process.PipId);
// Pip will read file source and file B.
// Two-phase cache will contain mapping
// wp => (#{source, B}, sp2), (#{source, A}, sp1)
File.WriteAllText(ArtifactToString(sourceFile), ArtifactToString(readFileB));
var result = RunScheduler().AssertCacheMiss(process.PipId);
var cacheInfo2 = result.RunData.ExecutionCachingInfos[process.PipId];
// Pip will read file source and file C.
// Two-phase cache will contain mappings
// wp => (#{source}, aug_marker), (#{source, B}, sp2), (#{source, A}, sp1)
// aug_wp1 => (#{source, C}, sp3)
//
// Fingerprint store contains mapping
// pip => (wp, #{source, C}, sp3)
File.WriteAllText(ArtifactToString(sourceFile), ArtifactToString(readFileC));
result = RunScheduler().AssertCacheMiss(process.PipId);
var cacheInfo3 = result.RunData.ExecutionCachingInfos[process.PipId];;
FingerprintStoreSession(
result,
store =>
{
store.TryGetFingerprintStoreEntryBySemiStableHash(process.FormattedSemiStableHash, out var entry);
// Weak fingerprint stored in the fingerprint store is the original one; not the augmented one.
// So use the weak fingerprint from previous build, i.e., cacheInfo2.
XAssert.AreEqual(cacheInfo2.WeakFingerprint.ToString(), entry.PipToFingerprintKeys.Value.WeakFingerprint);
XAssert.AreEqual(
JsonFingerprinter.ContentHashToString(cacheInfo3.PathSetHash),
entry.PipToFingerprintKeys.Value.FormattedPathSetHash);
XAssert.AreEqual(cacheInfo3.StrongFingerprint.ToString(), entry.PipToFingerprintKeys.Value.StrongFingerprint);
});
// Modify file C.
File.WriteAllText(ArtifactToString(readFileC), "modified");
// Runtime cache miss analysis will be done post cache look-up.
// Although perhaps the last evaluated cache entry is (#{source, A}, sp1), the fingerprint store
// will try to find the most relevant entry for cache miss analysis, which in this case (#{source, C}, sp3).
RunScheduler(m_testHooks).AssertCacheMiss(process.PipId);
XAssert.IsTrue(m_testHooks.FingerprintStoreTestHooks.TryGetCacheMiss(process.PipId, out var cacheMiss));
XAssert.AreEqual(CacheMissAnalysisResult.StrongFingerprintMismatch, cacheMiss.DetailAndResult.Result);
XAssert.IsTrue(cacheMiss.IsFromCacheLookUp);
// Ensure that the diff contains file C because the fingerprint store uses the most relevant cache entry for
// cache miss analysis, i.e., (#{source, C}, sp3)
XAssert.Contains(JsonConvert.SerializeObject(cacheMiss.DetailAndResult.Detail.Info).ToUpperInvariant(), ArtifactToString(readFileC).Replace("\\", "\\\\").ToUpperInvariant());
}
public void TestIpcPipWithVsoHashAndFileId()
{
// TODO
XAssert.IsTrue(true);
}
public void BatchingProcessTest()
{
EventListener.NestedLoggerHandler += eventData =>
{
if (eventData.EventId == (int)SharedLogEventId.CacheMissAnalysisBatchResults)
{
m_cacheMissAnalysisBatchList.Add(eventData.Payload.ToArray()[0].ToString());
}
};
m_cacheMissAnalysisBatchList = new List <string>();
var lenUnit = 100;
var string1 = new string('a', lenUnit);
var string2 = new string('b', 2 * lenUnit);
var string3 = new string('c', 3 * lenUnit);
var string4 = new string('d', 4 * lenUnit);
var string5 = new string('e', 3 * lenUnit);
var string6 = new string('f', 2 * lenUnit);
var string7 = new string('g', lenUnit);
var results = new List <JProperty>();
JProperty result1 = new JProperty("p1", new JObject(new JProperty("Result", string1)));
JProperty result2 = new JProperty("P2", new JObject(new JProperty("Result", string2)));
JProperty result3 = new JProperty("P3", new JObject(new JProperty("Result", string3)));
JProperty result4 = new JProperty("P4", new JObject(new JProperty("Result", string4)));
JProperty result5 = new JProperty("P5", new JObject(new JProperty("Result", string5)));
JProperty result6 = new JProperty("P6", new JObject(new JProperty("Result", string6)));
JProperty result7 = new JProperty("P7", new JObject(new JProperty("Result", string7)));
results.Add(result1);
results.Add(result2);
results.Add(result3);
results.Add(result4);
results.Add(result5);
results.Add(result6);
results.Add(result7);
var result1Len = result1.Name.Length + result1.Value.ToString().Length;
var result2Len = result2.Name.Length + result2.Value.ToString().Length;
var result3Len = result3.Name.Length + result3.Value.ToString().Length;
var result4Len = result4.Name.Length + result4.Value.ToString().Length;
var result5Len = result5.Name.Length + result5.Value.ToString().Length;
var result6Len = result6.Name.Length + result6.Value.ToString().Length;
var result7Len = result7.Name.Length + result7.Value.ToString().Length;
var timer = new Timer(o => { XAssert.IsTrue(false, "Process Timeout."); }, null, 10000, 10000);
// 1 batch per process
RuntimeCacheMissAnalyzer.s_numberOfBatchesLogged = 0;
Configuration.Logging.AriaIndividualMessageSizeLimitBytes = result1Len + result2Len + result3Len + result4Len + result5Len + result6Len + result7Len + 1;
RuntimeCacheMissAnalyzer.ProcessResults(results.ToArray(), Configuration, LoggingContext);
XAssert.AreEqual(m_cacheMissAnalysisBatchList.Count, 1, "Should have 1 batch logging.");
XAssert.Contains(m_cacheMissAnalysisBatchList[0], string1, string2, string3, string4, string5, string6, string7);
m_cacheMissAnalysisBatchList.Clear();
// 2 batch per process
RuntimeCacheMissAnalyzer.s_numberOfBatchesLogged = 0;
Configuration.Logging.AriaIndividualMessageSizeLimitBytes = result1Len + result2Len + result3Len + result4Len + 1;
RuntimeCacheMissAnalyzer.ProcessResults(results.ToArray(), Configuration, LoggingContext);
XAssert.AreEqual(m_cacheMissAnalysisBatchList.Count, 2, "Should have 2 batch logging.");
XAssert.Contains(m_cacheMissAnalysisBatchList[0], string1, string2, string3, string4);
XAssert.Contains(m_cacheMissAnalysisBatchList[1], string5, string6, string7);
m_cacheMissAnalysisBatchList.Clear();
// batch - single - batch in a process
RuntimeCacheMissAnalyzer.s_numberOfBatchesLogged = 0;
Configuration.Logging.AriaIndividualMessageSizeLimitBytes = result1Len + result2Len + result3Len + 20;
RuntimeCacheMissAnalyzer.ProcessResults(results.ToArray(), Configuration, LoggingContext);
XAssert.AreEqual(m_cacheMissAnalysisBatchList.Count, 3, "Should have 3 batch logging.");
XAssert.Contains(m_cacheMissAnalysisBatchList[0], string1, string2, string3);
XAssert.Contains(m_cacheMissAnalysisBatchList[1], string4);
XAssert.Contains(m_cacheMissAnalysisBatchList[2], string5, string6, string7);
XAssert.ContainsNot(m_cacheMissAnalysisBatchList[2], string4); // Make sure there no previous result in it
m_cacheMissAnalysisBatchList.Clear();
// batch - single - single - single - batch in a process
RuntimeCacheMissAnalyzer.s_numberOfBatchesLogged = 0;
Configuration.Logging.AriaIndividualMessageSizeLimitBytes = result1Len + result2Len + 1;
RuntimeCacheMissAnalyzer.ProcessResults(results.ToArray(), Configuration, LoggingContext);
XAssert.AreEqual(m_cacheMissAnalysisBatchList.Count, 5, "Should have 5 batch logging.");
XAssert.Contains(m_cacheMissAnalysisBatchList[0], string1, string2);
XAssert.Contains(m_cacheMissAnalysisBatchList[1], string3);
XAssert.Contains(m_cacheMissAnalysisBatchList[2], string4.Substring(string4.Length - Configuration.Logging.AriaIndividualMessageSizeLimitBytes / 2 + 20));
XAssert.ContainsNot(m_cacheMissAnalysisBatchList[2], string4);
XAssert.Contains(m_cacheMissAnalysisBatchList[2], "[...]");
XAssert.Contains(m_cacheMissAnalysisBatchList[3], string5);
XAssert.Contains(m_cacheMissAnalysisBatchList[4], string6, string7);
m_cacheMissAnalysisBatchList.Clear();
// all single in a process
RuntimeCacheMissAnalyzer.s_numberOfBatchesLogged = 0;
Configuration.Logging.AriaIndividualMessageSizeLimitBytes = result1Len + 1;
RuntimeCacheMissAnalyzer.ProcessResults(results.ToArray(), Configuration, LoggingContext);
XAssert.AreEqual(m_cacheMissAnalysisBatchList.Count, 7, "Should have 7 batch logging.");
XAssert.Contains(m_cacheMissAnalysisBatchList[0], string1);
XAssert.Contains(m_cacheMissAnalysisBatchList[1], string2.Substring(string2.Length - Configuration.Logging.AriaIndividualMessageSizeLimitBytes / 2 + 20));
XAssert.ContainsNot(m_cacheMissAnalysisBatchList[1], string2);
XAssert.Contains(m_cacheMissAnalysisBatchList[1], "[...]");
XAssert.Contains(m_cacheMissAnalysisBatchList[2], string3.Substring(string3.Length - Configuration.Logging.AriaIndividualMessageSizeLimitBytes / 2 + 20));
XAssert.ContainsNot(m_cacheMissAnalysisBatchList[2], string3);
XAssert.Contains(m_cacheMissAnalysisBatchList[2], "[...]");
XAssert.Contains(m_cacheMissAnalysisBatchList[3], string4.Substring(string4.Length - Configuration.Logging.AriaIndividualMessageSizeLimitBytes / 2 + 20));
XAssert.ContainsNot(m_cacheMissAnalysisBatchList[3], string4);
XAssert.Contains(m_cacheMissAnalysisBatchList[3], "[...]");
XAssert.Contains(m_cacheMissAnalysisBatchList[4], string5.Substring(string5.Length - Configuration.Logging.AriaIndividualMessageSizeLimitBytes / 2 + 20));
XAssert.ContainsNot(m_cacheMissAnalysisBatchList[4], string5);
XAssert.Contains(m_cacheMissAnalysisBatchList[4], "[...]");
XAssert.Contains(m_cacheMissAnalysisBatchList[5], string6.Substring(string6.Length - Configuration.Logging.AriaIndividualMessageSizeLimitBytes / 2 + 20));
XAssert.ContainsNot(m_cacheMissAnalysisBatchList[5], string6);
XAssert.Contains(m_cacheMissAnalysisBatchList[5], "[...]");
XAssert.Contains(m_cacheMissAnalysisBatchList[6], string7);
m_cacheMissAnalysisBatchList.Clear();
// all single in a process, test a result's len == maxLogLen
RuntimeCacheMissAnalyzer.s_numberOfBatchesLogged = 0;
Configuration.Logging.AriaIndividualMessageSizeLimitBytes = result3Len;
RuntimeCacheMissAnalyzer.ProcessResults(results.ToArray(), Configuration, LoggingContext);
XAssert.AreEqual(m_cacheMissAnalysisBatchList.Count, 7, "Should have 7 batch logging.");
XAssert.Contains(m_cacheMissAnalysisBatchList[0], string1);
XAssert.Contains(m_cacheMissAnalysisBatchList[1], string2);
XAssert.Contains(m_cacheMissAnalysisBatchList[2], string3.Substring(string4.Length - Configuration.Logging.AriaIndividualMessageSizeLimitBytes / 2 + 20));// Result has exact length as maxLogLen
XAssert.ContainsNot(m_cacheMissAnalysisBatchList[2], string3);
XAssert.Contains(m_cacheMissAnalysisBatchList[2], "[...]");
XAssert.Contains(m_cacheMissAnalysisBatchList[3], string4.Substring(string4.Length - Configuration.Logging.AriaIndividualMessageSizeLimitBytes / 2 + 20));
XAssert.ContainsNot(m_cacheMissAnalysisBatchList[3], string4);
XAssert.Contains(m_cacheMissAnalysisBatchList[3], "[...]");
XAssert.Contains(m_cacheMissAnalysisBatchList[4], string5.Substring(string4.Length - Configuration.Logging.AriaIndividualMessageSizeLimitBytes / 2 + 20));// Result has exact length as maxLogLen
XAssert.ContainsNot(m_cacheMissAnalysisBatchList[4], string5);
XAssert.Contains(m_cacheMissAnalysisBatchList[4], "[...]");
XAssert.Contains(m_cacheMissAnalysisBatchList[5], string6);
XAssert.Contains(m_cacheMissAnalysisBatchList[6], string7);
m_cacheMissAnalysisBatchList.Clear();
timer.Dispose();
}
/// <summary>
/// Verifies that the file output by a fragment exists in the resulting graph.
/// </summary>
/// <param name="graph">Resulting graph.</param>
/// <param name="fragmentOrigin">Graph fragment where the output originates.</param>
/// <param name="outputPath">Path to output file.</param>
private void VerifyProducerExists(PipGraph graph, TestPipGraphFragment fragmentOrigin, AbsolutePath outputPath)
{
var pipId = graph.TryGetProducer(FileArtifact.CreateOutputFile(RemapFragmentPath(fragmentOrigin, outputPath)));
XAssert.IsTrue(pipId.IsValid, $"Producer of '{outputPath.ToString(fragmentOrigin.Context.PathTable)}' from fragment '{fragmentOrigin.ModuleName}' could not be found in the resulting graph");
}
public void MarkExecuted(Pip executed)
{
bool added = m_executed.TryAdd(executed, Unit.Void);
XAssert.IsTrue(added, "Executed multiple times");
}
public void TestRedirectUserProfileDirectory()
{
// first run to create all necessary directories leading to obj directory
SetupTestData();
RunEngine();
string currentUserProfile = SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.UserProfile);
string junctionPath = Path.Combine(Configuration.Layout.ObjectDirectory.ToString(Context.PathTable), "buildXLUserProfile");
bool specialFolderInitializerWasCalled = false;
var translatedDirectory = new List <TranslateDirectoryData>();
var properties = new Dictionary <string, string>();
var expectedProperties = new Dictionary <string, string>
{
{ "APPDATA", Path.Combine(junctionPath, "AppData", "Roaming") },
{ "LOCALAPPDATA", Path.Combine(junctionPath, "AppData", "Local") },
{ "USERPROFILE", junctionPath },
{ "USERNAME", "buildXLUserProfile" },
{ "HOMEDRIVE", Path.GetPathRoot(junctionPath).TrimEnd('\\') },
{ "HOMEPATH", junctionPath.Substring(Path.GetPathRoot(junctionPath).TrimEnd('\\').Length) },
{ "INTERNETCACHE", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "INetCache") },
{ "INTERNETHISTORY", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "History") },
{ "INETCOOKIES", Path.Combine(junctionPath, "AppData", "Local", "Microsoft", "Windows", "INetCookies") },
{ "LOCALLOW", Path.Combine(junctionPath, "AppData", "LocalLow") },
};
// add the variables to the dictionary, so we can verify that the method overrides the existing values
foreach (var envVar in expectedProperties.Keys)
{
properties.Add(envVar, string.Empty);
}
try
{
var success = BuildXLEngine.RedirectUserProfileDirectory(
Configuration.Layout.ObjectDirectory,
translatedDirectory,
properties,
dict => { specialFolderInitializerWasCalled = true; },
true,
Context.PathTable,
LoggingContext);
// must have finished successfully
XAssert.IsTrue(success);
// verify the env block is properly populated
XAssert.AreSetsEqual(expectedProperties.Keys, properties.Keys, true);
XAssert.IsFalse(expectedProperties.Any(kvp => properties[kvp.Key] != kvp.Value));
XAssert.IsTrue(specialFolderInitializerWasCalled);
// verify junction
var openResult = FileUtilities.TryOpenDirectory(junctionPath, FileDesiredAccess.FileReadAttributes, FileShare.ReadWrite, FileFlagsAndAttributes.FileFlagOpenReparsePoint, out var handle);
XAssert.IsTrue(openResult.Succeeded);
using (handle)
{
var possibleTarget = FileUtilities.TryGetReparsePointTarget(handle, junctionPath);
XAssert.IsTrue(possibleTarget.Succeeded);
XAssert.AreEqual(FileSystemWin.NtPathPrefix + currentUserProfile, possibleTarget.Result);
}
// verify that we added a new directory translation
AbsolutePath.TryCreate(Context.PathTable, currentUserProfile, out var fromPath);
AbsolutePath.TryCreate(Context.PathTable, junctionPath, out var toPath);
XAssert.IsTrue(translatedDirectory.Count == 1);
XAssert.IsTrue(translatedDirectory[0].FromPath == fromPath && translatedDirectory[0].ToPath == toPath);
}
finally
{
// clean the junction after the test
var possibleProbe = FileUtilities.TryProbePathExistence(junctionPath, false);
if (possibleProbe.Succeeded && possibleProbe.Result != PathExistence.Nonexistent)
{
// we attempt to delete the junction, but we do not care if we failed to do
FileUtilities.TryRemoveDirectory(junctionPath, out int errCode);
}
}
}
public void RoundTripLog()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var pt = context.PathTable;
string path1 = A("c", "a", "b", "c");
var ap1 = AbsolutePath.Create(pt, path1);
string path2 = A("c", "d", "c", "a");
var ap2 = AbsolutePath.Create(pt, path2);
string path3 = A("d", "a", "c", "a");
var ap3 = AbsolutePath.Create(pt, path3);
string path3Caps = A("D", "A", "c", "a");
var ap3Caps = AbsolutePath.Create(pt, path3Caps);
int lastTestCase0EventIteration = 0;
int testCase0EventIteration = 0;
int expectedReadCount = 0;
Guid logId = Guid.NewGuid();
using (MemoryStream ms = new MemoryStream())
{
using (BinaryLogger writer = new BinaryLogger(ms, context, logId, lastStaticAbsolutePathIndex: ap2.Value.Value, closeStreamOnDispose: false))
{
using (var eventScope = writer.StartEvent((uint)EventId.TestCase0, workerId: 0))
{
eventScope.Writer.Write(ap1);
eventScope.Writer.Write("test string");
eventScope.Writer.Write(ap3);
eventScope.Writer.Write(12345);
expectedReadCount++;
lastTestCase0EventIteration++;
}
using (var eventScope = writer.StartEvent((uint)EventId.TestCase0, workerId: 0))
{
eventScope.Writer.Write("test string 2");
eventScope.Writer.Write(true);
eventScope.Writer.Write(ap3);
expectedReadCount++;
lastTestCase0EventIteration++;
}
}
ms.Position = 0;
using (BinaryLogReader reader = new BinaryLogReader(ms, context))
{
XAssert.IsTrue(reader.LogId.HasValue);
XAssert.AreEqual(logId, reader.LogId.Value);
reader.RegisterHandler((uint)EventId.TestCase0, (eventId, workerId, timestamp, eventReader) =>
{
switch (testCase0EventIteration)
{
case 0:
XAssert.AreEqual(ap1, eventReader.ReadAbsolutePath());
XAssert.AreEqual("test string", eventReader.ReadString());
XAssert.AreEqual(ap3, eventReader.ReadAbsolutePath());
XAssert.AreEqual(12345, eventReader.ReadInt32());
break;
case 1:
XAssert.AreEqual("test string 2", eventReader.ReadString());
XAssert.AreEqual(true, eventReader.ReadBoolean());
XAssert.AreEqual(ap3, eventReader.ReadAbsolutePath());
break;
default:
XAssert.Fail("Event raised unexpected number of times.");
break;
}
testCase0EventIteration++;
});
reader.RegisterHandler((uint)EventId.UnusedEvent, (eventId, workerId, timestamp, eventReader) =>
{
XAssert.Fail("This event should never be called.");
});
int readCount = 0;
BinaryLogReader.EventReadResult?readResult;
while ((readResult = reader.ReadEvent()) == BinaryLogReader.EventReadResult.Success)
{
XAssert.IsTrue(readCount < expectedReadCount);
readCount++;
}
XAssert.AreEqual(expectedReadCount, readCount);
XAssert.AreEqual(lastTestCase0EventIteration, testCase0EventIteration);
XAssert.AreEqual(BinaryLogReader.EventReadResult.EndOfStream, readResult);
}
}
}
public void RetryDeleteDirectoryContentsIfContentsPendingDelete()
{
try
{
// Need to disable POSIX delete to reproduce the Windows pending deletion state,
// which does not exist in POSIX
FileUtilities.PosixDeleteMode = PosixDeleteMode.NoRun;
string dir = Path.Combine(TemporaryDirectory, "dir");
Directory.CreateDirectory(dir);
string nestedFile = Path.Combine(dir, "nestedFile");
File.WriteAllText(nestedFile, "asdf");
SafeFileHandle nestedFileHandle;
FileUtilities.TryCreateOrOpenFile(
nestedFile,
FileDesiredAccess.GenericWrite,
FileShare.ReadWrite | FileShare.Delete,
FileMode.Open,
FileFlagsAndAttributes.None,
out nestedFileHandle);
// Hold open a handle to \file, but allow all file sharing permissions
using (nestedFileHandle)
{
// Sanity check that pending delete doesn't always return true
XAssert.IsFalse(FileUtilities.IsPendingDelete(nestedFileHandle));
Exception exception = null;
try
{
// Fails because of the open file that cannot be deleted
FileUtilities.DeleteDirectoryContents(dir, true);
}
catch (BuildXLException e)
{
exception = e;
XAssert.IsTrue(e.Message.StartsWith(FileUtilitiesMessages.DeleteDirectoryContentsFailed + NormalizeDirectoryPath(dir)));
// Rebuild the exception message using StringBuilder to handle breaklines
StringBuilder builder = new StringBuilder();
builder.AppendLine(NormalizeDirectoryPath(nestedFile));
builder.AppendLine(FileUtilitiesMessages.NoProcessesUsingHandle);
builder.AppendLine(FileUtilitiesMessages.PathMayBePendingDeletion);
XAssert.IsTrue(e.Message.Contains(builder.ToString()));
}
XAssert.IsTrue(exception != null);
// Check the open handle forced the file to be placed on the Windows pending deletion queue
XAssert.IsTrue(FileUtilities.IsPendingDelete(nestedFileHandle));
}
// After the file handle is closed, the delete goes through
XAssert.IsFalse(File.Exists(nestedFile));
// Check for retries and ERROR_DIR_NOT_EMPTY
AssertVerboseEventLogged(EventId.RetryOnFailureException, Helpers.DefaultNumberOfAttempts);
string logs = EventListener.GetLog();
var numMatches = Regex.Matches(logs, Regex.Escape("Native: RemoveDirectoryW for RemoveDirectory failed (0x91: The directory is not empty")).Count;
XAssert.AreEqual(Helpers.DefaultNumberOfAttempts, numMatches);
}
finally
{
// Re-enable POSIX delete for the remainder of tests
FileUtilities.PosixDeleteMode = PosixDeleteMode.RunFirst;
}
}
public void SnapshotStress()
{
var counters = new PipStateCounters();
bool[] exit = { false };
var transitionThreads = new Thread[4];
for (int i = 0; i < transitionThreads.Length; i++)
{
int threadId = i;
transitionThreads[i] = new Thread(
() =>
{
while (!Volatile.Read(ref exit[0]))
{
PipState[] orderedStates = s_pipStates.ToArray();
Shuffle(new Random(threadId), orderedStates);
// Create a new imaginary pip.
counters.AccumulateInitialState(PipState.Ignored, PipType.Process);
// Transition randomly until it is terminal.
var currentState = PipState.Ignored;
var priorStates = new HashSet <PipState> {
currentState
};
int nextCandidateStateIndex = 0;
while (!currentState.IsTerminal())
{
nextCandidateStateIndex = (nextCandidateStateIndex + 1) % orderedStates.Length;
PipState possibleNextState = orderedStates[nextCandidateStateIndex];
if (priorStates.Contains(possibleNextState))
{
// Cycle is possible.
break;
}
if (possibleNextState != currentState && currentState.CanTransitionTo(possibleNextState))
{
priorStates.Add(possibleNextState);
counters.AccumulateTransition(currentState, possibleNextState, PipType.Process);
currentState = possibleNextState;
}
}
}
});
}
try
{
foreach (Thread t in transitionThreads)
{
t.Start();
}
var snapshot = new PipStateCountersSnapshot();
long sum = 0;
while (sum < 100 * 1000)
{
counters.CollectSnapshot(s_pipTypes, snapshot);
foreach (PipState state in s_pipStates)
{
XAssert.IsTrue(snapshot[state] >= 0);
}
long newSum = s_pipStates.Sum(s => snapshot[s]);
XAssert.IsTrue(newSum >= sum, "Counters must be (probably non-strictly) monotonic");
sum = newSum;
}
}
finally
{
Volatile.Write(ref exit[0], true);
}
foreach (Thread t in transitionThreads)
{
t.Join();
}
}
public void FailToFindAllOpenHandlesPendingDeletion()
{
string dir = Path.Combine(TemporaryDirectory, "dir");
Directory.CreateDirectory(dir);
string file = Path.Combine(dir, "file");
File.WriteAllText(file, "asdf");
SafeFileHandle fileHandle;
FileUtilities.TryCreateOrOpenFile(
file,
FileDesiredAccess.GenericWrite,
FileShare.ReadWrite | FileShare.Delete,
FileMode.Open,
FileFlagsAndAttributes.None,
out fileHandle);
// Hold open a handle to \file, but allow all file sharing permissions
using (fileHandle)
{
XAssert.IsFalse(FileUtilities.IsPendingDelete(fileHandle));
// This will succeed without throwing an exception
File.Delete(file);
// But the open handle forces the file to be placed on a pending deletion queue
XAssert.IsTrue(FileUtilities.IsPendingDelete(fileHandle));
// Fail to detect open handles for files pending deletion
HashSet <string> deletedPaths = new HashSet <string>()
{
file
};
string openHandles = FileUtilities.FindAllOpenHandlesInDirectory(TemporaryDirectory, pathsPossiblyPendingDelete: deletedPaths);
// Rebuild the exception message using StringBuilder to handle breaklines
StringBuilder builder = new StringBuilder();
builder.AppendLine(file);
builder.AppendLine(FileUtilitiesMessages.NoProcessesUsingHandle);
builder.AppendLine(FileUtilitiesMessages.PathMayBePendingDeletion);
// Before Windows 10 Version 1903, attempting to create a file handle to a file pending deletion would throw an access exception, including calling File.Exists
// With Windows 10 Version 1903 and later, creating handles to files on the pending deletion queue does not throw exceptions and pending deletion files are considered deleted by File.Exists
// This change in behavior is NOT true for directories, see testing below for the directory behavior
XAssert.IsTrue(openHandles.Contains(builder.ToString()) || /* Check for Windows 10 Version 1903 and later */ !File.Exists(file));
XAssert.IsFalse(openHandles.Contains(FileUtilitiesMessages.ActiveHandleUsage + file));
}
XAssert.IsFalse(File.Exists(file));
SafeFileHandle dirHandle;
FileUtilities.TryCreateOrOpenFile(
dir,
FileDesiredAccess.GenericWrite,
FileShare.ReadWrite | FileShare.Delete,
FileMode.Open,
FileFlagsAndAttributes.FileFlagBackupSemantics,
out dirHandle);
// Hold open a handle to \dir, but allow all sharing permissions
using (dirHandle)
{
XAssert.IsFalse(FileUtilities.IsPendingDelete(dirHandle));
// This will succeed without throwing an exception
Directory.Delete(dir);
// But the open handle forces the directory to be placed on a pending deletion queue
XAssert.IsTrue(FileUtilities.IsPendingDelete(dirHandle));
// Fail to detect open handles for dirs pending deletion
HashSet <string> deletedPaths = new HashSet <string>()
{
dir
};
string openHandles = FileUtilities.FindAllOpenHandlesInDirectory(TemporaryDirectory, pathsPossiblyPendingDelete: deletedPaths);
// Rebuild the exception message using StringBuilder to handle breaklines
StringBuilder builder = new StringBuilder();
builder.AppendLine(dir);
builder.AppendLine(FileUtilitiesMessages.NoProcessesUsingHandle);
builder.AppendLine(FileUtilitiesMessages.PathMayBePendingDeletion);
XAssert.IsTrue(openHandles.Contains(builder.ToString()));
XAssert.IsFalse(openHandles.Contains(FileUtilitiesMessages.ActiveHandleUsage + dir));
}
XAssert.IsFalse(Directory.Exists(dir));
}
public void TestAddDirectoryToDropWithEmptyRelativePath()
{
string fakeDirectoryId = "123:1:12345";
var directoryPath = Path.Combine(TestOutputDirectory, "foo");
var files = new List <(string fileName, string remoteFileName)>
{
(Path.Combine(directoryPath, "a.txt"), "a.txt"),
(Path.Combine(directoryPath, "b.txt"), "b.txt"),
(Path.Combine(directoryPath, "bar", "c.txt"), "bar/c.txt"),
};
var dropPaths = new List <string>();
var expectedDropPaths = new HashSet <string>();
var regex = new Regex(".*", RegexOptions.IgnoreCase | RegexOptions.Compiled);
expectedDropPaths.AddRange(files.Where(a => regex.IsMatch(a.fileName)).Select(a => a.remoteFileName));
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
dropPaths.Add(item.RelativeDropPath);
return(Task.FromResult(AddFileResult.UploadedAndAssociated));
});
var ipcProvider = IpcFactory.GetProvider();
// this lambda mocks BuildXL server receiving 'GetSealedDirectoryContent' API call and returning a response
var ipcExecutor = new LambdaIpcOperationExecutor(op =>
{
var cmd = ReceiveGetSealedDirectoryContentCommandAndCheckItMatchesDirectoryId(op.Payload, fakeDirectoryId);
// Now 'fake' the response - here we only care about the 'FileName' field.
// In real life it's not the case, but this is a test and our custom addFileFunc
// in dropClient simply collects the drop file names.
var result = files.Select(a => CreateFakeSealedDirectoryFile(a.fileName)).ToList();
return(IpcResult.Success(cmd.RenderResult(result)));
});
WithIpcServer(
ipcProvider,
ipcExecutor,
new ServerConfig(),
(moniker, mockServer) =>
{
var bxlApiClient = CreateDummyBxlApiClient(ipcProvider, moniker);
WithSetup(
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath . --directoryFilter .*",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
XAssert.AreSetsEqual(expectedDropPaths, dropPaths, expectedResult: true);
},
bxlApiClient);
return(Task.CompletedTask);
}).GetAwaiter().GetResult();
}
public void DeleteDirectoryContentsLongPath()
{
string originalRoot = GetFullPath("testRoot");
// Create a directory with a path that's too long to normally delete
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 100; i++)
{
sb.Append("a");
}
string rootDir = @"\\?\" + originalRoot;
XAssert.IsTrue(CreateDirectoryW(rootDir, IntPtr.Zero));
rootDir = rootDir + "\\" + sb.ToString();
XAssert.IsTrue(CreateDirectoryW(rootDir, IntPtr.Zero));
rootDir = rootDir + "\\" + sb.ToString();
XAssert.IsTrue(CreateDirectoryW(rootDir, IntPtr.Zero));
// Write some files in the directory. Set their attributes as readonly to exercise the related logic in DeleteFile
string shorterPath = Path.Combine(originalRoot, "myFile.txt");
File.WriteAllText(shorterPath, "foo");
File.SetAttributes(shorterPath, FileAttributes.ReadOnly);
// And a file with a filename longer than maxpath
string longerPath = rootDir + @"\myFile.txt";
SafeFileHandle fileHandle;
var result = FileUtilities.TryCreateOrOpenFile(
longerPath,
FileDesiredAccess.GenericWrite,
FileShare.Delete,
FileMode.Create,
FileFlagsAndAttributes.FileAttributeNormal,
out fileHandle);
XAssert.IsTrue(result.Succeeded);
using (FileStream stream = new FileStream(fileHandle, FileAccess.Write))
{
stream.WriteByte(255);
}
FileUtilities.SetFileAttributes(longerPath, FileAttributes.ReadOnly);
string exeLink = Path.Combine(rootDir, "hardlink");
XAssert.IsTrue(CreateHardLinkIfSupported(link: exeLink, linkTarget: DummyWaiter.GetDummyWaiterExeLocation()));
// Add a hardlink. Perform deletion attempts while the hardlink target is in use
using (var waiter = DummyWaiter.RunAndWait())
{
// Attempt to delete with the managed API. This should fail because it contains nested paths that are too long
bool isPathTooLong = false;
try
{
Directory.Delete(rootDir, recursive: true);
}
catch (UnauthorizedAccessException)
{
// expected: when the long paths are supported, then
// CreateHardLinkW would be finished successfully,
// and directory deletion will fail because the executable is still running.
}
catch (PathTooLongException)
{
isPathTooLong = true;
}
// Expect failure only if long paths are not supported by the current .NET Framework version.
bool expectedIsPathTooLong = !LongPathsSupported;
XAssert.AreEqual(expectedIsPathTooLong, isPathTooLong, "Expected to encounter a PathTooLongException. If no exception is thrown by the System.IO method, this test isn't validating anything");
// Now use the native API. This should succeed
FileUtilities.DeleteDirectoryContents(originalRoot);
}
XAssert.IsTrue(Directory.Exists(originalRoot));
XAssert.AreEqual(0, Directory.GetFileSystemEntries(originalRoot).Length);
}
public async Task DynamicDirectoryMaterializationDoesNotDeleteDeclaredFiles()
{
var harness = CreateDefaultHarness();
harness.Seal();
var pathTable = harness.Environment.Context.PathTable;
DirectoryArtifact dynamicOutputDirectory = CreateDirectory();
FileArtifact explicitDeclaredOutputFile = CreateOutputFile(rootPath: dynamicOutputDirectory.Path, fileName: "expOut.txt");
string explicitNestedDeclaredOutputFileContents = "This is a nested declared file dependency";
var nestedDirectory = dynamicOutputDirectory.Path.Combine(pathTable, "nested");
FileArtifact explicitNestedDeclaredOutputFile = CreateOutputFile(rootPath: nestedDirectory, fileName: "expNestOut.txt");
string explicitDeclaredOutputFileContents = "This is a declared file dependency";
FileArtifact dynamicOutputFile = CreateOutputFile(rootPath: dynamicOutputDirectory.Path, fileName: "dynout.txt");
string dynamicOutputFileContents = "This is a dynamic file dependency";
await harness.StoreAndReportStringContent(explicitDeclaredOutputFileContents, explicitDeclaredOutputFile);
await harness.StoreAndReportStringContent(explicitNestedDeclaredOutputFileContents, explicitNestedDeclaredOutputFile);
await harness.StoreAndReportStringContent(dynamicOutputFileContents, dynamicOutputFile);
const string PriorOutputText = "Prior output";
harness.WriteText(dynamicOutputFile, PriorOutputText);
harness.WriteText(explicitDeclaredOutputFile, PriorOutputText);
harness.WriteText(explicitNestedDeclaredOutputFile, PriorOutputText);
// Add some extraneous files/directories to be removed
const string RemovedFileText = "Removed file text";
var removedNestedDirectory = dynamicOutputDirectory.Path.Combine(pathTable, "removedNested");
harness.WriteText(CreateOutputFile(rootPath: dynamicOutputDirectory.Path, fileName: "removedRootFile.txt"), RemovedFileText);
harness.WriteText(CreateOutputFile(rootPath: dynamicOutputDirectory.Path, fileName: "removedRootFile2.txt"), RemovedFileText);
harness.WriteText(CreateOutputFile(rootPath: nestedDirectory, fileName: "nestedremoved.txt"), RemovedFileText);
harness.WriteText(CreateOutputFile(rootPath: nestedDirectory, fileName: "nestedremoved1.txt"), RemovedFileText);
harness.WriteText(CreateOutputFile(rootPath: removedNestedDirectory, fileName: "removedNestedFile.txt"), RemovedFileText);
Directory.CreateDirectory(dynamicOutputDirectory.Path.Combine(pathTable, "removedEmptyNested").ToString(pathTable));
Directory.CreateDirectory(dynamicOutputDirectory.Path.Combine(pathTable, "removedEmptyNested3").ToString(pathTable));
harness.Environment.RegisterDynamicOutputDirectory(dynamicOutputDirectory);
var dynamicDirectoryContents = new[] { explicitDeclaredOutputFile, explicitNestedDeclaredOutputFile, dynamicOutputFile };
// Report files for directory
harness.FileContentManager.ReportDynamicDirectoryContents(
dynamicOutputDirectory,
dynamicDirectoryContents.Select(fa => FileArtifactWithAttributes.Create(fa, FileExistence.Required)),
PipOutputOrigin.NotMaterialized);
var producer = CreateCmdProcess(
dependencies: new FileArtifact[0],
outputs: new[] { explicitDeclaredOutputFile, explicitNestedDeclaredOutputFile },
directoryOutputs: new[] { dynamicOutputDirectory });
var fileConsumer = CreateCmdProcess(
dependencies: new[] { explicitDeclaredOutputFile, explicitNestedDeclaredOutputFile },
outputs: new[] { CreateOutputFile() });
var directoryConsumer = CreateCmdProcess(
dependencies: new FileArtifact[0],
directoryDependencies: new[] { dynamicOutputDirectory },
outputs: new[] { CreateOutputFile() });
// First materialize the files by materializing inputs for the pip which consumes the files directly
var fileMaterializationResult = await harness.FileContentManager.TryMaterializeDependenciesAsync(fileConsumer, harness.UntrackedOpContext);
Assert.True(fileMaterializationResult);
// Verify that only the explicit declared dependencies have their contents materialized
harness.VerifyContent(explicitDeclaredOutputFile, explicitDeclaredOutputFileContents);
harness.VerifyContent(explicitNestedDeclaredOutputFile, explicitNestedDeclaredOutputFileContents);
// The dynamic file shouldn't be materialized at this point
harness.VerifyContent(dynamicOutputFile, PriorOutputText);
// Modify the explicit outputs so that we can check later that materialization did not
// delete and re-materialize these files
const string ModifiedExplicitOutputText = "Modified explicit output";
harness.WriteText(explicitDeclaredOutputFile, ModifiedExplicitOutputText);
harness.WriteText(explicitNestedDeclaredOutputFile, ModifiedExplicitOutputText);
// Now materialize the directory
var directoryMaterializationResult = await harness.FileContentManager.TryMaterializeDependenciesAsync(directoryConsumer, harness.UntrackedOpContext);
Assert.True(directoryMaterializationResult);
var filesAfterMaterialization = new HashSet <string>(Directory.GetFiles(dynamicOutputDirectory.Path.ToString(pathTable), "*.*", SearchOption.AllDirectories), OperatingSystemHelper.PathComparer);
HashSet <string> dynamicDirectoryContentPaths = new HashSet <string>(OperatingSystemHelper.PathComparer);
dynamicDirectoryContentPaths.UnionWith(dynamicDirectoryContents.Select(f => f.Path.ToString(pathTable)));
// Check that the dynamic directory contents are the only remaining files
Assert.Subset(dynamicDirectoryContentPaths, filesAfterMaterialization);
Assert.Superset(dynamicDirectoryContentPaths, filesAfterMaterialization);
var directoriesAfterMaterialization = new HashSet <string>(Directory.GetDirectories(dynamicOutputDirectory.Path.ToString(pathTable), "*.*", SearchOption.AllDirectories), OperatingSystemHelper.PathComparer);
HashSet <string> dynamicDirectorySubDirectoryPaths = new HashSet <string>(OperatingSystemHelper.PathComparer);
dynamicDirectorySubDirectoryPaths.UnionWith(dynamicDirectoryContents.Select(f => f.Path.GetParent(pathTable).ToString(pathTable)));
// Don't count the root directory
dynamicDirectorySubDirectoryPaths.Remove(dynamicOutputDirectory.Path.ToString(pathTable));
// Check that the dynamic directory contents parent directories are the only remaining directories
Assert.Subset(dynamicDirectorySubDirectoryPaths, directoriesAfterMaterialization);
Assert.Superset(dynamicDirectorySubDirectoryPaths, directoriesAfterMaterialization);
XAssert.IsTrue(fileMaterializationResult);
// Verify all files have the expected contents
// Dynamic file should have the content for the cache
// Explicit dependencies should have the modified explicit content because
// the file content manager skips content which has been materialized already so
// the later modifications should go unnoticed
harness.VerifyContent(explicitDeclaredOutputFile, ModifiedExplicitOutputText);
harness.VerifyContent(explicitNestedDeclaredOutputFile, ModifiedExplicitOutputText);
harness.VerifyContent(dynamicOutputFile, dynamicOutputFileContents);
}
/// <summary>
/// Asserts that a file relative to <see cref="TemporaryStorageTestBase.TemporaryDirectory" /> exists.
/// </summary>
protected void AssertFileExists(string relPath)
{
string path = GetFullPath(relPath);
XAssert.IsTrue(File.Exists(path), "Expected path {0} to exist as a file", path);
}
private async Task ProcessWindowsCallHelper(
string functionName,
SandboxConfiguration config = null,
IEnumerable <string> extraDependencies = null,
IEnumerable <string> extraOutputs = null,
int callCount = 1,
string commandPrefix = "",
bool readsAndWritesDirectories = false,
bool untrackedOutputs = false,
string[] expectedWarningStrings = null,
string[] expectedErrorStrings = null)
{
if (config == null)
{
config = new SandboxConfiguration {
FileAccessIgnoreCodeCoverage = true, FailUnexpectedFileAccesses = true
};
}
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var fileContentTable = FileContentTable.CreateNew();
// have to force the config for truncation
config.OutputReportingMode = OutputReportingMode.FullOutputOnWarningOrError;
bool expectSuccess = expectedErrorStrings == null && expectedWarningStrings == null;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
string currentCodeFolder = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
Contract.Assume(currentCodeFolder != null);
string executable = Path.Combine(currentCodeFolder, DetourTestFolder, "DetoursTests.exe");
string workingDirectory = tempFiles.GetUniqueDirectory();
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
XAssert.IsTrue(File.Exists(executable), "Could not find the test file: " + executable);
FileArtifact executableFileArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
var extraUntrackedScopes = new List <AbsolutePath>();
var dependencies = new List <FileArtifact> {
executableFileArtifact
};
if (extraDependencies != null)
{
foreach (string file in extraDependencies)
{
string filePath = Path.Combine(workingDirectory, file);
AbsolutePath path = AbsolutePath.Create(pathTable, filePath);
if (readsAndWritesDirectories)
{
Directory.CreateDirectory(filePath);
// We don't support directories as inputs in BuildXL yet.
extraUntrackedScopes.Add(path);
}
else
{
File.WriteAllText(filePath, "Definitely a file");
FileArtifact fileArtifact = FileArtifact.CreateSourceFile(path);
dependencies.Add(fileArtifact);
}
}
}
var outputs = new List <FileArtifactWithAttributes>();
if (extraOutputs != null)
{
foreach (string file in extraOutputs)
{
string filePath = Path.Combine(workingDirectory, file);
AbsolutePath path = AbsolutePath.Create(pathTable, filePath);
if (readsAndWritesDirectories)
{
// We don't support directory outputs in BuildXL at the moment, so e.g. deleting a directory needs to be untracked.
extraUntrackedScopes.Add(path);
}
else if (untrackedOutputs)
{
extraUntrackedScopes.Add(path);
}
else
{
FileArtifact fileArtifact = FileArtifact.CreateSourceFile(path).CreateNextWrittenVersion();
outputs.Add(fileArtifact.WithAttributes());
}
}
}
var tempDirectory = tempFiles.GetUniqueDirectory();
var environmentVariables = new List <EnvironmentVariable>();
var environmentValue = new PipDataBuilder(pathTable.StringTable);
var tempPath = AbsolutePath.Create(pathTable, tempDirectory);
environmentValue.Add(tempPath);
environmentVariables.Add(new EnvironmentVariable(StringId.Create(pathTable.StringTable, "TMP"), environmentValue.ToPipData(" ", PipDataFragmentEscaping.NoEscaping)));
environmentVariables.Add(new EnvironmentVariable(StringId.Create(pathTable.StringTable, "TEMP"), environmentValue.ToPipData(" ", PipDataFragmentEscaping.NoEscaping)));
var untrackedPaths = CmdHelper.GetCmdDependencies(pathTable);
var untrackedScopes = extraUntrackedScopes.Concat(CmdHelper.GetCmdDependencyScopes(pathTable).Concat(new[] { tempPath })).Distinct();
var pip = new Process(
executableFileArtifact,
workingDirectoryAbsolutePath,
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.NoEscaping, commandPrefix + functionName + "Logging"),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .From(environmentVariables),
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .From(dependencies),
ReadOnlyArray <FileArtifactWithAttributes> .From(outputs),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(untrackedPaths),
ReadOnlyArray <AbsolutePath> .From(untrackedScopes),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
if (expectSuccess)
{
await AssertProcessSucceedsAsync(
context,
config,
pip);
}
else
{
await AssertProcessCompletesWithStatus(
SandboxedProcessPipExecutionStatus.ExecutionFailed,
context,
config,
pip,
null);
}
}
int expectedErrorCount = 0;
int expectedWarningCount = 0;
IEnumerable <string> requiredLogMessageSubstrings = new string[] { };
if (expectedErrorStrings != null)
{
expectedErrorCount = expectedErrorStrings.Count();
requiredLogMessageSubstrings = requiredLogMessageSubstrings.Concat(expectedErrorStrings);
}
if (expectedWarningStrings != null)
{
expectedWarningCount = expectedWarningStrings.Count();
requiredLogMessageSubstrings = requiredLogMessageSubstrings.Concat(expectedWarningStrings);
}
SetExpectedFailures(expectedErrorCount, expectedWarningCount, requiredLogMessageSubstrings.ToArray());
}
public void ChildProcessCanBreakawayWhenConfigured(bool letInfiniteWaiterSurvive)
{
// Skip this test if running on .NET Framework with vstest
// Reason: when this is the case and code coverage is turned on, launching breakaway
// processes here causes the code coverage monitoring process to hang.
if (!OperatingSystemHelper.IsDotNetCore && IsRunningInVsTestTestHost())
{
return;
}
// We use InfiniteWaiter (a process that waits forever) as a long-living process that we can actually check it can
// escape the job object
var fam = new FileAccessManifest(
Context.PathTable,
childProcessesToBreakawayFromSandbox: letInfiniteWaiterSurvive ? new[] { InfiniteWaiterToolName } : null)
{
FailUnexpectedFileAccesses = false
};
// We instruct the regular test process to spawn InfiniteWaiter as a child
var info = ToProcessInfo(
ToProcess(
Operation.SpawnExe(
Context.PathTable,
CreateFileArtifactWithName(InfiniteWaiterToolName, TestDeploymentDir))),
fileAccessManifest: fam);
// Let's shorten the default time to wait for nested processes, since we are spawning
// a process that never ends and we don't want this test to wait for that long
info.NestedProcessTerminationTimeout = TimeSpan.FromMilliseconds(10);
var result = RunProcess(info).GetAwaiter().GetResult();
XAssert.AreEqual(0, result.ExitCode);
if (!letInfiniteWaiterSurvive)
{
// If we didn't let infinite waiter escape, we should have killed it when the job object was finalized
XAssert.IsTrue(result.Killed);
XAssert.Contains(
result.SurvivingChildProcesses.Select(p => p?.Path).Where(p => p != null).Select(p => System.IO.Path.GetFileName(p).ToUpperInvariant()),
InfiniteWaiterToolName.ToUpperInvariant());
}
else
{
// If we did let it escape, then nothing should have been killed (nor tried to survive and later killed, from the job object point of view)
XAssert.IsFalse(result.Killed);
if (result.SurvivingChildProcesses != null && result.SurvivingChildProcesses.Any())
{
var survivors = string.Join(
", ",
result.SurvivingChildProcesses.Select(p => p?.Path != null ? System.IO.Path.GetFileName(p.Path) : "<unknown>"));
XAssert.Fail($"Unexpected {result.SurvivingChildProcesses.Count()} surviving child processes: {survivors}");
}
// Let's retrieve the child process and confirm it survived
var infiniteWaiterInfo = RetrieveChildProcessesCreatedBySpawnExe(result).Single();
// The fact that this does not throw confirms survival
var dummyWaiter = Process.GetProcessById(infiniteWaiterInfo.pid);
// Just being protective, let's make sure we are talking about the same process
XAssert.AreEqual(infiniteWaiterInfo.processName, dummyWaiter.ProcessName);
// Now let's kill the surviving process, since we don't want it to linger around unnecessarily
dummyWaiter.Kill();
}
}