public void DoubleWriteReportedForPathOrderedBefore()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, DoubleWritePath);
Process producer = graph.AddProcess(producerOutput);
Process violator = graph.AddProcess(violatorOutput);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.ReadWrite, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.DoubleWrite,
FileMonitoringViolationAnalyzer.AccessLevel.Write,
producerOutput,
violator,
producer),
"The violator is after the producer, so this should be a double-write on the produced path.");
analyzer.AssertNoExtraViolationsCollected();
AssertErrorEventLogged(EventId.FileMonitoringError);
}
public void DoubleWriteNotReportedForPathOrderedAfter()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, DoubleWritePath);
Process violator = graph.AddProcess(violatorOutput);
Process producer = graph.AddProcess(producerOutput);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.ReadWrite, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null,
ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .Empty,
out _);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.UndeclaredOutput,
FileMonitoringViolationAnalyzer.AccessLevel.Write,
producerOutput,
violator,
null),
"The violator has an undeclared output but it wasn't reported.");
analyzer.AssertNoExtraViolationsCollected();
AssertErrorEventLogged(EventId.FileMonitoringError);
}
public void UndeclaredOrderedReadReportedForPathOrderedBefore()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph, true);
AbsolutePath producerOutput = CreateAbsolutePath(context, ProducedPath);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
Process producer = graph.AddProcess(producerOutput);
Process violator = graph.AddProcess(violatorOutput);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.Read, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null,
ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .Empty,
out _);
AssertVerboseEventLogged(LogEventId.DependencyViolationUndeclaredOrderedRead);
AssertErrorEventLogged(EventId.FileMonitoringError);
}
public void DoubleWriteEventLoggedOnViolationError()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new FileMonitoringViolationAnalyzer(LoggingContext, context, graph, new QueryableEmptyFileContentManager(), validateDistribution: false, ignoreDynamicWritesOnAbsentProbes: false, unexpectedFileAccessesAsErrors: true);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, DoubleWritePath);
Process producer = graph.AddProcess(producerOutput);
Process violator = graph.AddProcess(violatorOutput);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.ReadWrite, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null,
ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .Empty,
out _);
AssertVerboseEventLogged(LogEventId.DependencyViolationDoubleWrite);
AssertErrorEventLogged(EventId.FileMonitoringError);
}
public void InvalidAccessedPathExcludedFromAnalysis()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
var declaredOuput = CreateAbsolutePath(context, X("/X/out/declared"));
var process = graph.AddProcess(declaredOuput);
var result = analyzer.AnalyzePipViolations(
process,
new[]
{
CreateViolationForInvalidPath(RequestedAccess.Read, X("c/invalid:name_1.txt"))
},
new[]
{
CreateViolationForInvalidPath(RequestedAccess.Read, X("c/invalid:name_2.txt"))
},
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null
);
XAssert.IsTrue(result.IsViolationClean);
}
public void TestMalformedPaths()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var translations = new[]
{
CreateInputTranslation(pathTable, new string[] { "K", "dbs", "sh", "dtb", "b" }, new string[] { "d", "dbs", "sh", "dtb", "0629_120346" }),
};
var translator = new DirectoryTranslator();
translator.AddTranslations(translations, pathTable);
translator.Seal();
// None of these paths should be mutated by DirectoryTranslator
foreach (string pathToTest in new string[] {
@"\??\",
@"\\?\",
@":",
@"d",
@"k",
@"",
})
{
// Test edge cases for various paths. Note these explicitly don't go through path generation utilities because they can
// massage away malformed paths that we explicitly want to test.
AssertAreEqual(pathToTest, translator.Translate(pathToTest));
}
}
public void UndeclaredReadCycleReportedForPathOrderedAfter()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, ProducedPath);
Process violator = graph.AddProcess(violatorOutput);
Process producer = graph.AddProcess(producerOutput);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.Read, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.UndeclaredReadCycle,
FileMonitoringViolationAnalyzer.AccessLevel.Read,
producerOutput,
violator,
producer),
"An UndeclaredReadCycle should have been reported since an earlier pip has an undeclared read of a later pip's output.");
analyzer.AssertNoExtraViolationsCollected();
AssertErrorEventLogged(EventId.FileMonitoringError);
}
public Harness(string testOutputDirectory)
{
Context = BuildXLContext.CreateInstanceForTesting();
var config = ConfigurationHelpers.GetDefaultForTesting(Context.PathTable, AbsolutePath.Create(Context.PathTable, System.IO.Path.Combine(testOutputDirectory, "config.dc")));
m_env = new DummyPipExecutionEnvironment(
CreateLoggingContextForTest(),
Context,
config,
subst: FileUtilities.TryGetSubstSourceAndTarget(testOutputDirectory, out var substSource, out var substTarget)
? (substSource, substTarget)
: default((string, string)?),
sandboxConnection: GetSandboxConnection());
var sealContentsCache = new ConcurrentBigMap <DirectoryArtifact, int[]>();
Table = Context.PathTable;
// Create the paths in the PathTable before creating the CachedFileSystemView
Path(a);
Path(b);
Path(c);
Path(d);
Path(e);
Path(f);
Path(g);
Path(h);
Path(i);
Path(j);
Path(k);
Path(l);
m_fileSystem = new PipFileSystemView();
m_fileSystem.Initialize(Table);
}
public async Task ProcessInContainerGeneratingNestedOutputsPassesAllSandboxValidations()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"outputs\");
var outputFileNested = tempFiles.GetUniqueFileName(@"outputs\nested\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var outputFileNestedPath = AbsolutePath.Create(pathTable, outputFileNested);
FileUtilities.CreateDirectory(outputFileNestedPath.GetParent(pathTable).ToString(pathTable));
// Arguments to create two output files, one nested under the other
var arguments = $"echo hi > {outputFile} && echo bye > {outputFileNested}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFileNestedPath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
var pipExecutionResult = await RunProcess(context, pip);
// Observe that the fact the execution result succeeds means that the expected outputs are in their expected place
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
}
}
public void ReadRaceEventLoggedOnViolationDistributed()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new FileMonitoringViolationAnalyzer(LoggingContext, context, graph, new QueryableEmptyFileContentManager(), validateDistribution: true, ignoreDynamicWritesOnAbsentProbes: false, unexpectedFileAccessesAsErrors: true);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, ProducedPath);
Process producer = graph.AddProcess(producerOutput);
Process violator = graph.AddProcess(violatorOutput);
graph.SetConcurrentRange(producer, violator);
AnalyzePipViolationsResult analyzePipViolationsResult =
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.Read, producerOutput) },
null, // whitelisted accesses
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null);
AssertTrue(!analyzePipViolationsResult.IsViolationClean);
AssertErrorEventLogged(EventId.FileMonitoringError);
AssertVerboseEventLogged(LogEventId.DependencyViolationReadRace);
}
public void UndeclaredOutput()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
AbsolutePath declaredOutput = CreateAbsolutePath(context, X("/X/out/produced1"));
AbsolutePath undeclaredOutput = CreateAbsolutePath(context, X("/X/out/produced2"));
Process producer = graph.AddProcess(declaredOutput);
analyzer.AnalyzePipViolations(
producer,
new[]
{
CreateViolation(RequestedAccess.Write, undeclaredOutput),
},
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.UndeclaredOutput,
FileMonitoringViolationAnalyzer.AccessLevel.Write,
undeclaredOutput,
producer,
null),
"The violator has an undeclared output but it wasn't reported.");
AssertErrorEventLogged(EventId.FileMonitoringError);
}
public async Task TombstoneFileDoesNotRepresentARequiredOutput()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName();
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var renamedFile = $"{outputFile}.renamed";
var renamedFilePath = AbsolutePath.Create(pathTable, renamedFile);
// Arguments to create an output file that gets immediately renamed.
// However, both files are marked as required, even though the original one
// is not there after the rename happens
var arguments = $"echo hi > {outputFile} && move {outputFile} {renamedFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(renamedFilePath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
// the move command under the console seems to have some issue with the detours policy
// This is orthogonal to the test, and we don't care about detours at this point
var pipExecutionResult = await RunProcess(context, pip, failUnexpectedFileAccesses : false);
// The redirected output is created as a tombstone file, but the sandboxed pip executor should report it as an absent file
AssertErrorEventLogged(ProcessesLogEventId.PipProcessMissingExpectedOutputOnCleanExit);
AssertErrorEventLogged(ProcessesLogEventId.PipProcessExpectedMissingOutputs);
}
}
public CacheCoreFingerprintStoreTests(ITestOutputHelper output)
: base(output)
{
Context = BuildXLContext.CreateInstanceForTesting();
FingerprintStore = new CacheCoreFingerprintStore(Session);
ContentCache = new CacheCoreArtifactContentCache(Session, rootTranslator: null);
}
public void TestDirectoryTranslatorUsedForJunctionsInCloudBuild()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var translations = new[]
{
CreateInputTranslation(pathTable, new string[] { "K", "dbs", "sh", "dtb", "b" }, new string[] { "d", "dbs", "sh", "dtb", "0629_120346" }),
CreateInputTranslation(pathTable, new string[] { "d", "dbs", "sh", "dtb", "0629_120346", "Build" }, new string[] { "d", "dbs", "el", "dtb", "Build" }),
CreateInputTranslation(pathTable, new string[] { "d", "dbs", "sh", "dtb", "0629_120346", "Target" }, new string[] { "d", "dbs", "el", "dtb", "Target" })
};
string error;
XAssert.IsTrue(DirectoryTranslator.ValidateDirectoryTranslation(pathTable, translations, out error));
var translator = new DirectoryTranslator();
translator.AddTranslations(translations, pathTable);
translator.Seal();
AssertEqualTranslatedPath(translator, pathTable, new string[] { "d", "dbs", "el", "dtb", "Build", "x64", "debug", "perl.cmd" }, new string[] { "K", "dbs", "sh", "dtb", "b", "Build", "x64", "debug", "perl.cmd" });
AssertEqualTranslatedPath(translator, pathTable, new string[] { "d", "dbs", "el", "dtb", "Target", "x64", "debug", "perl.cmd" }, new string[] { "K", "dbs", "sh", "dtb", "b", "Target", "x64", "debug", "perl.cmd" });
AssertEqualTranslatedPath(translator, pathTable, new string[] { "d", "dbs", "sh", "dtb", "0629_120346" }, new string[] { "K", "dbs", "sh", "dtb", "b" });
AssertEqualTranslatedPath(translator, new string[] { @"\\?\d", "dbs", "el", "dtb", "Build", "x64", "debug", "perl.cmd" }, new string[] { @"\\?\K", "dbs", "sh", "dtb", "b", "Build", "x64", "debug", "perl.cmd" });
AssertEqualTranslatedPath(translator, new string[] { @"\??\d", "dbs", "el", "dtb", "Build", "x64", "debug", "perl.cmd" }, new string[] { @"\??\K", "dbs", "sh", "dtb", "b", "Build", "x64", "debug", "perl.cmd" });
}
public async Task RoundtripSimpleStructure()
{
var context = BuildXLContext.CreateInstanceForTesting();
var cache = new InMemoryArtifactContentCache(context);
var originalEntry = new PipCacheDescriptorV2Metadata()
{
Id = 123,
NumberOfWarnings = 456,
TraceInfo = "Good job."
};
Possible <ContentHash> maybeStored =
await cache.TrySerializeAndStoreContent(originalEntry);
XAssert.IsTrue(maybeStored.Succeeded);
var maybeDeserialized =
await cache.TryLoadAndDeserializeContent <PipCacheDescriptorV2Metadata>(maybeStored.Result);
XAssert.IsTrue(maybeDeserialized.Succeeded);
PipCacheDescriptorV2Metadata roundtripped = maybeDeserialized.Result;
XAssert.IsNotNull(roundtripped, "Expected content available");
XAssert.AreEqual(originalEntry.Id, roundtripped.Id);
XAssert.AreEqual(originalEntry.NumberOfWarnings, roundtripped.NumberOfWarnings);
XAssert.AreEqual(originalEntry.TraceInfo, roundtripped.TraceInfo);
}
public void TestDirectoryTranslatorEnvironmentInjection()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var someDir = X("/c/some/dir");
var anotherDir = X("/c/another/dir");
var differentDir = X("/d/different/dir");
var differentVolume = X("/e/different/volume");
var translations = new[]
{
new DirectoryTranslator.Translation(someDir, anotherDir),
new DirectoryTranslator.Translation(differentDir, differentVolume)
};
var environmentVariable = DirectoryTranslator.GetEnvironmentVaribleRepresentationForTranslations(translations);
XAssert.AreEqual(DirectoryTranslator.TranslatedDirectoriesEnvironmentVariable, environmentVariable.variable);
XAssert.AreEqual(environmentVariable.value, $"{someDir}|{anotherDir};{differentDir}|{differentVolume}");
var translator = new DirectoryTranslator();
translator.AddDirectoryTranslationFromEnvironment(environmentVariable.value);
// The directory translator calls EnsureDirectoryPath() when adding translations, adding the directory seperator char, lets remove
// it so SequenceEqual can be used conviniently
var sanitizedTranslations = translator.Translations.Select(t =>
new DirectoryTranslator.Translation(t.SourcePath.Substring(0, t.SourcePath.Length - 1), t.TargetPath.Substring(0, t.TargetPath.Length - 1)));
XAssert.IsTrue(Enumerable.SequenceEqual(sanitizedTranslations, translations));
}
private FrontEndHostController CreateHost()
{
var factory = new FrontEndFactory();
factory.AddFrontEnd(new DummyFrontEnd1());
factory.TrySeal(new LoggingContext("UnitTest"));
var controller = new FrontEndHostController(factory, new DScriptWorkspaceResolverFactory(), new EvaluationScheduler(degreeOfParallelism: 8), collectMemoryAsSoonAsPossible: false);
var context = BuildXLContext.CreateInstanceForTesting();
var fileSystem = new InMemoryFileSystem(context.PathTable);
((IFrontEndController)controller).InitializeHost(
new FrontEndContext(context, new LoggingContext("UnitTest"), fileSystem),
new ConfigurationImpl()
{
FrontEnd = new FrontEndConfiguration()
{
MaxFrontEndConcurrency = 1,
}
});
var inMemoryCache = new EngineCache(
new InMemoryArtifactContentCache(context),
new InMemoryTwoPhaseFingerprintStore());
controller.InitializeInternalForTesting(
Task.FromResult(new Possible <EngineCache>(inMemoryCache)),
AbsolutePath.Create(context.PathTable, TestOutputDirectory));
return(controller);
}
public void ReadLevelViolationReportedForUnknownPath()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
AbsolutePath unknown = CreateAbsolutePath(context, X("/X/out/unknown"));
AbsolutePath produced = CreateAbsolutePath(context, ProducedPath);
Process process = graph.AddProcess(produced);
analyzer.AnalyzePipViolations(
process,
new[] { CreateViolation(RequestedAccess.Read, unknown) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.MissingSourceDependency,
FileMonitoringViolationAnalyzer.AccessLevel.Read,
unknown,
process,
null),
"A MissingSourceDependency should have been reported with no suggested value");
analyzer.AssertNoExtraViolationsCollected();
AssertErrorEventLogged(EventId.FileMonitoringError);
}
private CompositeGraphFingerprint GenerateRandomTopLevelHash(string configPath, string index, bool flag)
{
var context1 = BuildXLContext.CreateInstanceForTesting();
var configuration1 = ConfigurationHelpers.GetDefaultForTesting(context1.PathTable, AbsolutePath.Create(context1.PathTable, configPath));
var evaluationFilter1 = new EvaluationFilter(
context1.SymbolTable,
context1.PathTable,
new FullSymbol[0],
new[]
{
AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"testFile{index}.txt")),
},
CollectionUtilities.EmptyArray <StringId>());
configuration1.Layout.ObjectDirectory = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"ObjectDirectory{index}"));
configuration1.Layout.TempDirectory = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"TempDirectory{index}"));
configuration1.Layout.SourceDirectory = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"SourceDirectory{index}"));
configuration1.Logging.SubstTarget = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"SubstTarget{index}"));
configuration1.Engine.CompressGraphFiles = flag;
configuration1.Schedule.SkipHashSourceFile = flag;
configuration1.Schedule.ComputePipStaticFingerprints = flag;
var loggingContext1 = CreateLoggingContextForTest();
var fileContentTable1 = FileContentTable.CreateNew(loggingContext1);
return(GraphFingerprinter.TryComputeFingerprint(loggingContext1, configuration1.Startup, configuration1, context1.PathTable, evaluationFilter1, fileContentTable1, "111aaa", null).ExactFingerprint);
}
public async Task TestSerialization()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var symbolTable = new SymbolTable(pathTable.StringTable);
var whitelist = new FileAccessWhitelist(context);
var path1 = AbsolutePath.Create(pathTable, @"\\fakePath\foo.txt");
var path2 = AbsolutePath.Create(pathTable, @"\\fakePath\bar.txt");
var regex1 = new SerializableRegex(@"dir\foo.txt");
var executableEntry1 = new ExecutablePathWhitelistEntry(
path1, regex1, true, "entry1");
var executableEntry2 = new ExecutablePathWhitelistEntry(
path2, new SerializableRegex("bar"), false, "entry2");
whitelist.Add(executableEntry1);
whitelist.Add(executableEntry2);
var symbol1 = FullSymbol.Create(symbolTable, "symbol1");
var valueEntry = new ValuePathFileAccessWhitelistEntry(
symbol1, new SerializableRegex("symbol1"), false, null);
var symbol2 = FullSymbol.Create(symbolTable, "symbol2");
var valueEntry2 = new ValuePathFileAccessWhitelistEntry(
symbol2, new SerializableRegex("symbol2"), false, "entry4");
whitelist.Add(valueEntry);
whitelist.Add(valueEntry2);
XAssert.AreEqual(3, whitelist.UncacheableEntryCount);
XAssert.AreEqual(1, whitelist.CacheableEntryCount);
XAssert.AreEqual("Unnamed", valueEntry.Name);
using (var ms = new MemoryStream())
{
BuildXLWriter writer = new BuildXLWriter(true, ms, true, true);
whitelist.Serialize(writer);
ms.Position = 0;
BuildXLReader reader = new BuildXLReader(true, ms, true);
var deserialized = await FileAccessWhitelist.DeserializeAsync(reader, Task.FromResult <PipExecutionContext>(context));
XAssert.AreEqual(2, deserialized.ExecutablePathEntries.Count);
XAssert.AreEqual(1, deserialized.ExecutablePathEntries[path1].Count);
XAssert.AreEqual(true, deserialized.ExecutablePathEntries[path1][0].AllowsCaching);
XAssert.AreEqual(regex1.ToString(), deserialized.ExecutablePathEntries[path1][0].PathRegex.ToString());
XAssert.AreEqual(executableEntry1.Name, deserialized.ExecutablePathEntries[path1][0].Name);
XAssert.AreEqual(executableEntry2.Name, deserialized.ExecutablePathEntries[path2][0].Name);
XAssert.AreEqual(2, deserialized.ValuePathEntries.Count);
XAssert.AreEqual(1, deserialized.ValuePathEntries[symbol1].Count);
XAssert.AreEqual(false, deserialized.ValuePathEntries[symbol1][0].AllowsCaching);
XAssert.AreEqual(valueEntry.Name, deserialized.ValuePathEntries[symbol1][0].Name);
XAssert.AreEqual(valueEntry2.Name, deserialized.ValuePathEntries[symbol2][0].Name);
XAssert.AreEqual(3, deserialized.UncacheableEntryCount);
XAssert.AreEqual(1, deserialized.CacheableEntryCount);
}
}
protected PipTestBase(ITestOutputHelper output) : base(output)
{
Context = BuildXLContext.CreateInstanceForTesting();
PathTable.DebugPathTable = Context.PathTable;
TestBinRoot = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(System.Reflection.Assembly.GetExecutingAssembly()));
TestBinRootPath = AbsolutePath.Create(Context.PathTable, TestBinRoot);
SourceRoot = Path.Combine(TemporaryDirectory, SourceRootPrefix);
SourceRootPath = AbsolutePath.Create(Context.PathTable, SourceRoot);
CmdExecutable = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, CmdHelper.OsShellExe));
string testProcessFolder = Path.Combine(TestBinRoot, "TestProcess");
string platformDir = Dispatch.CurrentOS().ToString();
string exe = Path.Combine(testProcessFolder, platformDir, TestProcessToolName);
TestProcessExecutable = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, exe));
// Test process depends on C://Windows (when running on Windows)
TestProcessDependencies = OperatingSystemHelper.IsUnixOS
? new AbsolutePath[0]
: new AbsolutePath[] { AbsolutePath.Create(Context.PathTable, Environment.GetFolderPath(Environment.SpecialFolder.Windows)) };
ObjectRoot = Path.Combine(TemporaryDirectory, ObjectRootPrefix);
ObjectRootPath = AbsolutePath.Create(Context.PathTable, ObjectRoot);
CacheRoot = Path.Combine(TemporaryDirectory, CacheRootPrefix);
BaseSetup();
}
private static async Task <bool> CreateAndRunPip(
PipProgram program,
string tempDirectory,
string outFile,
IEnumerable <string> restInstructions,
bool is64Bit)
{
Contract.Requires(restInstructions != null);
Contract.Requires(tempDirectory != null);
Contract.Requires(!string.IsNullOrEmpty(outFile));
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
using (var fileAccessListener = new FileAccessListener(Events.Log))
{
fileAccessListener.RegisterEventSource(BuildXL.Processes.ETWLogger.Log);
var fileContentTable = FileContentTable.CreateNew();
var config = ConfigurationHelpers.GetDefaultForTesting(context.PathTable, AbsolutePath.Create(context.PathTable, Path.Combine(tempDirectory, "config.dc")));
config.Sandbox.LogObservedFileAccesses = true;
Pip pip = null;
var instructions = restInstructions as string[] ?? restInstructions.ToArray();
switch (program)
{
case PipProgram.Cmd:
pip = CreateCmdPip(context, tempDirectory, outFile, is64Bit);
break;
case PipProgram.Self:
pip = CreateSelfPip(context, tempDirectory, outFile, instructions, is64Bit);
break;
}
Contract.Assume(pip != null);
PipResult executeResult = await Execute(context, fileContentTable, config, pip);
bool valid = false;
switch (program)
{
case PipProgram.Cmd:
valid = ValidateCmd(fileAccessListener.FileAccesses, outFile, is64Bit);
break;
case PipProgram.Self:
valid = ValidateSelf(
fileAccessListener.FileAccesses,
instructions.Length > 0 ? instructions[0] : string.Empty,
outFile,
is64Bit);
break;
}
return(executeResult.Status == PipResultStatus.Succeeded && valid);
}
}
/// <summary>
/// Initialize the test
/// </summary>
public DirectoryMembershipFingerprinterTests(ITestOutputHelper output) : base(output)
{
m_context = BuildXLContext.CreateInstanceForTesting();
m_pipTable = new PipTable(m_context.PathTable, m_context.SymbolTable, initialBufferSize: 1024, maxDegreeOfParallelism: 1, debug: true);
m_mountInfo = new MountPathExpander(m_context.PathTable);
RegisterEventSource(global::BuildXL.Scheduler.ETWLogger.Log);
}
public PerProcessPipPerformanceInformationStoreTests(ITestOutputHelper output)
: base(output)
{
m_context = BuildXLContext.CreateInstanceForTesting();
m_configuration = ConfigurationHelpers.GetDefaultForTesting(m_context.PathTable, AbsolutePath.Create(m_context.PathTable, Path.Combine(TemporaryDirectory, "config.ds")));
m_perPipPerformanceInformationStore = new PerProcessPipPerformanceInformationStore(m_configuration.Logging.MaxNumPipTelemetryBatches, m_configuration.Logging.AriaIndividualMessageSizeLimitBytes);
}
public async Task CorrelateMoveFileAsync()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
AbsolutePath sourceDirectory = tempFiles.GetDirectory(pathTable, "Source");
AbsolutePath sourceFile = sourceDirectory.Combine(pathTable, "SourceFile.txt");
var destinationFile = tempFiles.GetFileName(pathTable, "DestinationFile.txt");
WriteFile(pathTable, sourceFile, "content");
var process = CreateDetourProcess(
context,
pathTable,
tempFiles,
argumentStr: "CorrelateMoveFile",
inputFiles: ReadOnlyArray <FileArtifact> .Empty,
inputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
outputFiles: ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
FileArtifactWithAttributes.Create(
FileArtifact.CreateOutputFile(destinationFile), FileExistence.Required)),
outputDirectories: ReadOnlyArray <DirectoryArtifact> .Empty,
untrackedScopes: ReadOnlyArray <AbsolutePath> .FromWithoutCopy(sourceDirectory));
var correlator = new Correlator(pathTable);
SandboxedProcessPipExecutionResult result = await RunProcessAsync(
pathTable : pathTable,
ignoreSetFileInformationByHandle : false,
ignoreZwRenameFileInformation : false,
monitorNtCreate : true,
ignoreReparsePoints : false,
ignoreNonCreateFileReparsePoints : false,
monitorZwCreateOpenQueryFile : false,
context : context,
pip : process,
detoursListener : correlator,
errorString : out _);
VerifyNormalSuccess(context, result);
XAssert.IsTrue(File.Exists(destinationFile.ToString(pathTable)));
var toVerify = new List <(AbsolutePath, RequestedAccess, FileAccessStatus)>
{
(sourceFile, RequestedAccess.ReadWrite, FileAccessStatus.Allowed),
(destinationFile, RequestedAccess.Write, FileAccessStatus.Allowed)
};
VerifyFileAccesses(context, result.AllReportedFileAccesses, toVerify.ToArray());
correlator.VerifyCorrelation(new Correlator.VerifiedCorrelation(
destinationFile.ToString(pathTable),
ReportedFileOperation.MoveFileWithProgressDest,
sourceFile.ToString(pathTable),
ReportedFileOperation.MoveFileWithProgressSource));
}
}
private async Task <SandboxedProcessResult> RunWithPluginAsync(
bool shimAllProcesses,
bool shouldBeShimmed,
bool shouldFindMatch,
string[] processMatches,
StringBuilder stdOutSb,
StringBuilder stdErrSb,
string shimmedText = null)
{
var context = BuildXLContext.CreateInstanceForTesting();
var shimProgramPath = GetShimProgramPath(context);
var pluginDlls = GetPluginDlls(context);
string executable = CmdHelper.CmdX64;
var fam = CreateCommonFileAccessManifest(context.PathTable);
fam.SubstituteProcessExecutionInfo = new SubstituteProcessExecutionInfo(
shimProgramPath,
shimAllProcesses: shimAllProcesses,
processMatches: processMatches.Select(pm => new ShimProcessMatch(PathAtom.Create(context.StringTable, pm), PathAtom.Invalid)).ToArray())
{
SubstituteProcessExecutionPluginDll32Path = pluginDlls.x86Dll,
SubstituteProcessExecutionPluginDll64Path = pluginDlls.x64Dll
};
string childOutput = GetChildOutputForPluginTest(shouldBeShimmed, shimmedText);
string childArgs = $"{executable} /D /C echo {childOutput}";
string args = $"/D /C echo Top-level cmd. Running child process && {childArgs}";
SandboxedProcessInfo sandboxedProcessInfo = CreateCommonSandboxedProcessInfo(
context,
executable,
args,
fam,
stdOutSb,
stdErrSb);
ISandboxedProcess sandboxedProcess =
await SandboxedProcessFactory.StartAsync(sandboxedProcessInfo, forceSandboxing : true)
.ConfigureAwait(false);
SandboxedProcessResult result = await sandboxedProcess.GetResultAsync().ConfigureAwait(false);
if (shouldFindMatch || processMatches.Length == 0)
{
// Plugin should be called when a match is found or no process match is specified.
// When plugin is entered, expect to see "Entering CommandMatches" text in the log.
// CODESYNC: Public\Src\Engine\UnitTests\Processes.TestPrograms\SubstituteProcessExecutionPlugin\dllmain.cpp
AssertLogContains(true, "Entering CommandMatches");
}
return(result);
}
public async Task IsolationLevelControlsWriteRedirection(ContainerIsolationLevel containerIsolationLevel)
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"fileOutputs\");
var opaqueOutputFile = tempFiles.GetUniqueFileName(@"directoryOutputs\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var opaqueOutputFilePath = AbsolutePath.Create(pathTable, opaqueOutputFile);
var arguments = $"echo hi > {outputFile} && echo bye > {opaqueOutputFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
});
var opaqueOutputs = ReadOnlyArray <DirectoryArtifact> .FromWithoutCopy(
new[]
{
new DirectoryArtifact(opaqueOutputFilePath.GetParent(pathTable), 1, isSharedOpaque: containerIsolationLevel == ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories),
}
);
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, opaqueOutputs, containerIsolationLevel);
var pipExecutionResult = await RunProcess(context, pip);
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
var redirectedDirForFiles = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "fileOutputs");
var redirectedFile = redirectedDirForFiles.Combine(pathTable, outputFilePath.GetName(pathTable)).ToString(pathTable);
var redirectedDirForDirectories = pip.UniqueRedirectedDirectoryRoot.Combine(pathTable, "directoryOutputs");
var redirectedOpaqueFile = redirectedDirForDirectories.Combine(pathTable, opaqueOutputFilePath.GetName(pathTable)).ToString(pathTable);
// Make sure outputs got redirected based on the configured isolation level
switch (containerIsolationLevel)
{
case ContainerIsolationLevel.IsolateOutputFiles:
XAssert.IsTrue(File.Exists(redirectedFile));
break;
case ContainerIsolationLevel.IsolateExclusiveOpaqueOutputDirectories:
case ContainerIsolationLevel.IsolateSharedOpaqueOutputDirectories:
XAssert.IsTrue(File.Exists(redirectedOpaqueFile));
break;
}
}
}
public async Task DeserializationReturnsNullIfUnavailable()
{
var context = BuildXLContext.CreateInstanceForTesting();
var cache = new InMemoryArtifactContentCache(context);
ContentHash imaginaryContent = ContentHashingUtilities.HashBytes(Encoding.UTF8.GetBytes("Imagination"));
var maybeDeserialized =
await cache.TryLoadAndDeserializeContent <PipCacheDescriptorV2Metadata>(imaginaryContent);
XAssert.IsTrue(maybeDeserialized.Succeeded);
XAssert.IsNull(maybeDeserialized.Result, "Should be a miss (cache empty)");
}
public void DoubleWritePolicyDeterminesViolationSeverity(DoubleWritePolicy doubleWritePolicy)
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(
LoggingContext,
context,
graph,
// Set this to test the logic of base.HandleDependencyViolation(...) instead of the overriding fake
doLogging: true,
collectNonErrorViolations: true);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, DoubleWritePath);
Process producer = graph.AddProcess(producerOutput, doubleWritePolicy);
Process violator = graph.AddProcess(violatorOutput, doubleWritePolicy);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.ReadWrite, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null,
ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .Empty,
out _);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.DoubleWrite,
FileMonitoringViolationAnalyzer.AccessLevel.Write,
producerOutput,
violator,
producer),
"The violator is after the producer, so this should be a double-write on the produced path.");
analyzer.AssertNoExtraViolationsCollected();
AssertVerboseEventLogged(LogEventId.DependencyViolationDoubleWrite);
// Based on the double write policy, the violation is an error or a warning
if (doubleWritePolicy == DoubleWritePolicy.DoubleWritesAreErrors)
{
AssertErrorEventLogged(EventId.FileMonitoringError);
}
else
{
AssertWarningEventLogged(EventId.FileMonitoringWarning);
}
}
/// <summary>
/// Creates an instance of <see cref="TestPipGraphFragment"/>.
/// </summary>
public TestPipGraphFragment(LoggingContext loggingContext, string sourceRoot, string objectRoot, string redirectedRoot, string moduleName, bool useTopSort = false)
{
Contract.Requires(loggingContext != null);
Contract.Requires(!string.IsNullOrEmpty(sourceRoot));
Contract.Requires(!string.IsNullOrEmpty(objectRoot));
Contract.Requires(!string.IsNullOrEmpty(moduleName));
Context = BuildXLContext.CreateInstanceForTesting();
m_loggingContext = loggingContext;
m_sourceRoot = AbsolutePath.Create(Context.PathTable, sourceRoot);
m_objectRoot = AbsolutePath.Create(Context.PathTable, objectRoot);
m_expander = new MountPathExpander(Context.PathTable);
var configuration = new ConfigurationImpl()
{
Schedule =
{
UseFixedApiServerMoniker = true,
ComputePipStaticFingerprints = true,
}
};
m_useTopSort = useTopSort;
PipGraph = m_useTopSort
? new PipGraphFragmentBuilderTopSort(Context, configuration, m_expander)
: new PipGraphFragmentBuilder(Context, configuration, m_expander);
ModuleName = moduleName;
var specFileName = moduleName + ".dsc";
m_specPath = m_sourceRoot.Combine(Context.PathTable, specFileName);
m_moduleId = ModuleId.Create(StringId.Create(Context.StringTable, moduleName));
var modulePip = ModulePip.CreateForTesting(
Context.StringTable,
m_specPath,
m_moduleId);
PipGraph.AddModule(modulePip);
PipGraph.AddSpecFile(new SpecFilePip(new FileArtifact(m_specPath), new LocationData(m_specPath, 0, 0), modulePip.Module));
m_defaultConstructionHelper = PipConstructionHelper.CreateForTesting(
Context,
objectRoot: m_objectRoot,
redirectedRoot: AbsolutePath.Create(Context.PathTable, redirectedRoot),
pipGraph: PipGraph,
moduleName: moduleName,
specRelativePath: Path.Combine(m_sourceRoot.GetName(Context.PathTable).ToString(Context.StringTable), specFileName),
specPath: m_specPath,
symbol: moduleName + "_defaultValue");
}
