private RootFilter ParseFilter(IEnumerable <string> values, string commandLineFilter, string defaultFilter, out SymbolTable symbolTable, out PathTable pathTable)
{
var loggingContext = new LoggingContext("EngineScheduleTests.ParseFilter");
pathTable = new PathTable();
// EngineSchedulTest operate on the real filesystem for now
var fileSystem = new PassThroughFileSystem(pathTable);
BuildXLContext context = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
symbolTable = context.SymbolTable;
var config = new CommandLineConfiguration
{
Filter = commandLineFilter,
Startup =
{
ImplicitFilters = new List <string>(values ?? Enumerable.Empty <string>()),
},
Engine =
{
DefaultFilter = defaultFilter,
}
};
RootFilter rootFilter;
XAssert.IsTrue(EngineSchedule.TryGetPipFilter(loggingContext, context, config, config, TryGetPathByMountName,
rootFilter: out rootFilter));
return(rootFilter);
}
public void DoubleWriteNotReportedForPathOrderedAfter()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, DoubleWritePath);
Process violator = graph.AddProcess(violatorOutput);
Process producer = graph.AddProcess(producerOutput);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.ReadWrite, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null,
ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .Empty,
out _);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.UndeclaredOutput,
FileMonitoringViolationAnalyzer.AccessLevel.Write,
producerOutput,
violator,
null),
"The violator has an undeclared output but it wasn't reported.");
analyzer.AssertNoExtraViolationsCollected();
AssertErrorEventLogged(EventId.FileMonitoringError);
}
public void UndeclaredOrderedReadReportedForPathOrderedBefore()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph, true);
AbsolutePath producerOutput = CreateAbsolutePath(context, ProducedPath);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
Process producer = graph.AddProcess(producerOutput);
Process violator = graph.AddProcess(violatorOutput);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.Read, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null,
ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .Empty,
out _);
AssertVerboseEventLogged(LogEventId.DependencyViolationUndeclaredOrderedRead);
AssertErrorEventLogged(EventId.FileMonitoringError);
}
public void DoubleWriteEventLoggedOnViolationError()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new FileMonitoringViolationAnalyzer(LoggingContext, context, graph, new QueryableEmptyFileContentManager(), validateDistribution: false, ignoreDynamicWritesOnAbsentProbes: false, unexpectedFileAccessesAsErrors: true);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, DoubleWritePath);
Process producer = graph.AddProcess(producerOutput);
Process violator = graph.AddProcess(violatorOutput);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.ReadWrite, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null,
ReadOnlyArray <(FileArtifact, FileMaterializationInfo, PipOutputOrigin)> .Empty,
out _);
AssertVerboseEventLogged(LogEventId.DependencyViolationDoubleWrite);
AssertErrorEventLogged(EventId.FileMonitoringError);
}
private void RunGraphFingerprintValueTest(string[] expectedValues, string commandLineFilter, string[] commandLineValues, bool expectSuccess = true)
{
var pathTable = new PathTable();
// EngineSchedulTest operate on the real filesystem for now
var fileSystem = new PassThroughFileSystem(pathTable);
BuildXLContext context = EngineContext.CreateNew(CancellationToken.None, pathTable, fileSystem);
var config = new CommandLineConfiguration
{
Filter = commandLineFilter,
Startup = { ImplicitFilters = new List <string>(commandLineValues) },
Engine = { DefaultFilter = null }
};
EvaluationFilter evaluationFilter;
if (!expectSuccess)
{
XAssert.IsFalse(EngineSchedule.TryGetEvaluationFilter(BuildXL.TestUtilities.Xunit.XunitBuildXLTest.CreateLoggingContextForTest(), context, config, config, out evaluationFilter));
}
else
{
XAssert.IsTrue(EngineSchedule.TryGetEvaluationFilter(BuildXL.TestUtilities.Xunit.XunitBuildXLTest.CreateLoggingContextForTest(), context, config, config, out evaluationFilter));
foreach (FullSymbol symbol in evaluationFilter.ValueNamesToResolve)
{
string value = symbol.ToString(context.SymbolTable);
XAssert.IsTrue(expectedValues.Contains(value));
}
XAssert.AreEqual(expectedValues.Length, evaluationFilter.ValueNamesToResolve.Count());
}
}
public CacheCoreFingerprintStoreTests(ITestOutputHelper output)
: base(output)
{
Context = BuildXLContext.CreateInstanceForTesting();
FingerprintStore = new CacheCoreFingerprintStore(Session);
ContentCache = new CacheCoreArtifactContentCache(Session, rootTranslator: null);
}
public Harness(string testOutputDirectory)
{
Context = BuildXLContext.CreateInstanceForTesting();
var config = ConfigurationHelpers.GetDefaultForTesting(Context.PathTable, AbsolutePath.Create(Context.PathTable, System.IO.Path.Combine(testOutputDirectory, "config.dc")));
m_env = new DummyPipExecutionEnvironment(
CreateLoggingContextForTest(),
Context,
config,
subst: FileUtilities.TryGetSubstSourceAndTarget(testOutputDirectory, out var substSource, out var substTarget)
? (substSource, substTarget)
: default((string, string)?),
sandboxConnection: GetSandboxConnection());
var sealContentsCache = new ConcurrentBigMap <DirectoryArtifact, int[]>();
Table = Context.PathTable;
// Create the paths in the PathTable before creating the CachedFileSystemView
Path(a);
Path(b);
Path(c);
Path(d);
Path(e);
Path(f);
Path(g);
Path(h);
Path(i);
Path(j);
Path(k);
Path(l);
m_fileSystem = new PipFileSystemView();
m_fileSystem.Initialize(Table);
}
public async Task ProcessInContainerGeneratingNestedOutputsPassesAllSandboxValidations()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName(@"outputs\");
var outputFileNested = tempFiles.GetUniqueFileName(@"outputs\nested\");
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var outputFileNestedPath = AbsolutePath.Create(pathTable, outputFileNested);
FileUtilities.CreateDirectory(outputFileNestedPath.GetParent(pathTable).ToString(pathTable));
// Arguments to create two output files, one nested under the other
var arguments = $"echo hi > {outputFile} && echo bye > {outputFileNested}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFileNestedPath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
var pipExecutionResult = await RunProcess(context, pip);
// Observe that the fact the execution result succeeds means that the expected outputs are in their expected place
XAssert.AreEqual(SandboxedProcessPipExecutionStatus.Succeeded, pipExecutionResult.Status);
}
}
public async Task TombstoneFileDoesNotRepresentARequiredOutput()
{
var context = BuildXLContext.CreateInstanceForTesting();
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var pathTable = context.PathTable;
var outputFile = tempFiles.GetUniqueFileName();
var outputFilePath = AbsolutePath.Create(pathTable, outputFile);
var renamedFile = $"{outputFile}.renamed";
var renamedFilePath = AbsolutePath.Create(pathTable, renamedFile);
// Arguments to create an output file that gets immediately renamed.
// However, both files are marked as required, even though the original one
// is not there after the rename happens
var arguments = $"echo hi > {outputFile} && move {outputFile} {renamedFile}";
var outputs = ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(
new[]
{
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(outputFilePath), FileExistence.Required),
FileArtifactWithAttributes.Create(FileArtifact.CreateOutputFile(renamedFilePath), FileExistence.Required)
});
var pip = CreateConsoleProcessInContainer(context, tempFiles, pathTable, arguments, outputs, CollectionUtilities.EmptyArray <DirectoryArtifact>().ToReadOnlyArray());
// the move command under the console seems to have some issue with the detours policy
// This is orthogonal to the test, and we don't care about detours at this point
var pipExecutionResult = await RunProcess(context, pip, failUnexpectedFileAccesses : false);
// The redirected output is created as a tombstone file, but the sandboxed pip executor should report it as an absent file
AssertErrorEventLogged(ProcessesLogEventId.PipProcessMissingExpectedOutputOnCleanExit);
AssertErrorEventLogged(ProcessesLogEventId.PipProcessExpectedMissingOutputs);
}
}
private Task <SandboxedProcessPipExecutionResult> RunProcess(BuildXLContext context, Process pip, bool failUnexpectedFileAccesses = true)
{
var loggingContext = CreateLoggingContextForTest();
var pipExecutor = new SandboxedProcessPipExecutor(
context,
loggingContext,
pip,
new SandboxConfiguration {
FailUnexpectedFileAccesses = failUnexpectedFileAccesses
},
layoutConfig: null,
loggingConfig: null,
rootMappings: new Dictionary <string, string>(),
processInContainerManager: new ProcessInContainerManager(loggingContext, context.PathTable),
whitelist: null,
makeInputPrivate: null,
makeOutputPrivate: null,
semanticPathExpander: SemanticPathExpander.Default,
disableConHostSharing: false,
pipEnvironment: new PipEnvironment(loggingContext),
validateDistribution: false,
isLazySharedOpaqueOutputDeletionEnabled: false,
tempDirectoryCleaner: new TestMoveDeleteCleaner(TestOutputDirectory),
directoryArtifactContext: TestDirectoryArtifactContext.Empty);
return(pipExecutor.RunAsync());
}
protected PipTestBase(ITestOutputHelper output) : base(output)
{
Context = BuildXLContext.CreateInstanceForTesting();
PathTable.DebugPathTable = Context.PathTable;
TestBinRoot = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(System.Reflection.Assembly.GetExecutingAssembly()));
TestBinRootPath = AbsolutePath.Create(Context.PathTable, TestBinRoot);
SourceRoot = Path.Combine(TemporaryDirectory, SourceRootPrefix);
SourceRootPath = AbsolutePath.Create(Context.PathTable, SourceRoot);
CmdExecutable = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, CmdHelper.OsShellExe));
string testProcessFolder = Path.Combine(TestBinRoot, "TestProcess");
string platformDir = Dispatch.CurrentOS().ToString();
string exe = Path.Combine(testProcessFolder, platformDir, TestProcessToolName);
TestProcessExecutable = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, exe));
// Test process depends on C://Windows (when running on Windows)
TestProcessDependencies = OperatingSystemHelper.IsUnixOS
? new AbsolutePath[0]
: new AbsolutePath[] { AbsolutePath.Create(Context.PathTable, Environment.GetFolderPath(Environment.SpecialFolder.Windows)) };
ObjectRoot = Path.Combine(TemporaryDirectory, ObjectRootPrefix);
ObjectRootPath = AbsolutePath.Create(Context.PathTable, ObjectRoot);
CacheRoot = Path.Combine(TemporaryDirectory, CacheRootPrefix);
BaseSetup();
}
private static void AssertEqualRelativePaths(
BuildXLContext context,
string objPath,
Remapper remapper,
RelativePath expectedPath,
RelativePath actualPath)
{
if (remapper?.PathAtomStringRemapper != null)
{
var expectedAtoms = expectedPath.GetAtoms();
var actualAtoms = actualPath.GetAtoms();
XAssert.AreEqual(
expectedAtoms.Length,
actualAtoms.Length,
$"{nameof(AbsolutePath)} values don't match for objPath: {objPath}");
for (int i = 0; i < expectedAtoms.Length; ++i)
{
AssertEqualPathAtoms(context, objPath, remapper, expectedAtoms[i], actualAtoms[i]);
}
}
else
{
XAssert.AreEqual(expectedPath, actualPath, $"{nameof(RelativePath)} values don't match for objPath: {objPath}");
}
}
public void ReadRaceEventLoggedOnViolationDistributed()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new FileMonitoringViolationAnalyzer(LoggingContext, context, graph, new QueryableEmptyFileContentManager(), validateDistribution: true, ignoreDynamicWritesOnAbsentProbes: false, unexpectedFileAccessesAsErrors: true);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, ProducedPath);
Process producer = graph.AddProcess(producerOutput);
Process violator = graph.AddProcess(violatorOutput);
graph.SetConcurrentRange(producer, violator);
AnalyzePipViolationsResult analyzePipViolationsResult =
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.Read, producerOutput) },
null, // whitelisted accesses
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null);
AssertTrue(!analyzePipViolationsResult.IsViolationClean);
AssertErrorEventLogged(EventId.FileMonitoringError);
AssertVerboseEventLogged(LogEventId.DependencyViolationReadRace);
}
public void InvalidAccessedPathExcludedFromAnalysis()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
var declaredOuput = CreateAbsolutePath(context, X("/X/out/declared"));
var process = graph.AddProcess(declaredOuput);
var result = analyzer.AnalyzePipViolations(
process,
new[]
{
CreateViolationForInvalidPath(RequestedAccess.Read, X("c/invalid:name_1.txt"))
},
new[]
{
CreateViolationForInvalidPath(RequestedAccess.Read, X("c/invalid:name_2.txt"))
},
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null
);
XAssert.IsTrue(result.IsViolationClean);
}
public async Task RoundtripSimpleStructure()
{
var context = BuildXLContext.CreateInstanceForTesting();
var cache = new InMemoryArtifactContentCache(context);
var originalEntry = new PipCacheDescriptorV2Metadata()
{
Id = 123,
NumberOfWarnings = 456,
TraceInfo = "Good job."
};
Possible <ContentHash> maybeStored =
await cache.TrySerializeAndStoreContent(originalEntry);
XAssert.IsTrue(maybeStored.Succeeded);
var maybeDeserialized =
await cache.TryLoadAndDeserializeContent <PipCacheDescriptorV2Metadata>(maybeStored.Result);
XAssert.IsTrue(maybeDeserialized.Succeeded);
PipCacheDescriptorV2Metadata roundtripped = maybeDeserialized.Result;
XAssert.IsNotNull(roundtripped, "Expected content available");
XAssert.AreEqual(originalEntry.Id, roundtripped.Id);
XAssert.AreEqual(originalEntry.NumberOfWarnings, roundtripped.NumberOfWarnings);
XAssert.AreEqual(originalEntry.TraceInfo, roundtripped.TraceInfo);
}
public void DoubleWriteReportedForPathOrderedBefore()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, DoubleWritePath);
Process producer = graph.AddProcess(producerOutput);
Process violator = graph.AddProcess(violatorOutput);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.ReadWrite, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.DoubleWrite,
FileMonitoringViolationAnalyzer.AccessLevel.Write,
producerOutput,
violator,
producer),
"The violator is after the producer, so this should be a double-write on the produced path.");
analyzer.AssertNoExtraViolationsCollected();
AssertErrorEventLogged(EventId.FileMonitoringError);
}
public void TestDirectoryTranslatorEnvironmentInjection()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var someDir = X("/c/some/dir");
var anotherDir = X("/c/another/dir");
var differentDir = X("/d/different/dir");
var differentVolume = X("/e/different/volume");
var translations = new[]
{
new DirectoryTranslator.Translation(someDir, anotherDir),
new DirectoryTranslator.Translation(differentDir, differentVolume)
};
var environmentVariable = DirectoryTranslator.GetEnvironmentVaribleRepresentationForTranslations(translations);
XAssert.AreEqual(DirectoryTranslator.TranslatedDirectoriesEnvironmentVariable, environmentVariable.variable);
XAssert.AreEqual(environmentVariable.value, $"{someDir}|{anotherDir};{differentDir}|{differentVolume}");
var translator = new DirectoryTranslator();
translator.AddDirectoryTranslationFromEnvironment(environmentVariable.value);
// The directory translator calls EnsureDirectoryPath() when adding translations, adding the directory seperator char, lets remove
// it so SequenceEqual can be used conviniently
var sanitizedTranslations = translator.Translations.Select(t =>
new DirectoryTranslator.Translation(t.SourcePath.Substring(0, t.SourcePath.Length - 1), t.TargetPath.Substring(0, t.TargetPath.Length - 1)));
XAssert.IsTrue(Enumerable.SequenceEqual(sanitizedTranslations, translations));
}
private FrontEndHostController CreateHost()
{
var factory = new FrontEndFactory();
factory.AddFrontEnd(new DummyFrontEnd1());
factory.TrySeal(new LoggingContext("UnitTest"));
var controller = new FrontEndHostController(factory, new DScriptWorkspaceResolverFactory(), new EvaluationScheduler(degreeOfParallelism: 8), collectMemoryAsSoonAsPossible: false);
var context = BuildXLContext.CreateInstanceForTesting();
var fileSystem = new InMemoryFileSystem(context.PathTable);
((IFrontEndController)controller).InitializeHost(
new FrontEndContext(context, new LoggingContext("UnitTest"), fileSystem),
new ConfigurationImpl()
{
FrontEnd = new FrontEndConfiguration()
{
MaxFrontEndConcurrency = 1,
}
});
var inMemoryCache = new EngineCache(
new InMemoryArtifactContentCache(context),
new InMemoryTwoPhaseFingerprintStore());
controller.InitializeInternalForTesting(
Task.FromResult(new Possible <EngineCache>(inMemoryCache)),
AbsolutePath.Create(context.PathTable, TestOutputDirectory));
return(controller);
}
public void TestMalformedPaths()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var translations = new[]
{
CreateInputTranslation(pathTable, new string[] { "K", "dbs", "sh", "dtb", "b" }, new string[] { "d", "dbs", "sh", "dtb", "0629_120346" }),
};
var translator = new DirectoryTranslator();
translator.AddTranslations(translations, pathTable);
translator.Seal();
// None of these paths should be mutated by DirectoryTranslator
foreach (string pathToTest in new string[] {
@"\??\",
@"\\?\",
@":",
@"d",
@"k",
@"",
})
{
// Test edge cases for various paths. Note these explicitly don't go through path generation utilities because they can
// massage away malformed paths that we explicitly want to test.
AssertAreEqual(pathToTest, translator.Translate(pathToTest));
}
}
private CompositeGraphFingerprint GenerateRandomTopLevelHash(string configPath, string index, bool flag)
{
var context1 = BuildXLContext.CreateInstanceForTesting();
var configuration1 = ConfigurationHelpers.GetDefaultForTesting(context1.PathTable, AbsolutePath.Create(context1.PathTable, configPath));
var evaluationFilter1 = new EvaluationFilter(
context1.SymbolTable,
context1.PathTable,
new FullSymbol[0],
new[]
{
AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"testFile{index}.txt")),
},
CollectionUtilities.EmptyArray <StringId>());
configuration1.Layout.ObjectDirectory = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"ObjectDirectory{index}"));
configuration1.Layout.TempDirectory = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"TempDirectory{index}"));
configuration1.Layout.SourceDirectory = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"SourceDirectory{index}"));
configuration1.Logging.SubstTarget = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"SubstTarget{index}"));
configuration1.Engine.CompressGraphFiles = flag;
configuration1.Schedule.SkipHashSourceFile = flag;
configuration1.Schedule.ComputePipStaticFingerprints = flag;
var loggingContext1 = CreateLoggingContextForTest();
var fileContentTable1 = FileContentTable.CreateNew(loggingContext1);
return(GraphFingerprinter.TryComputeFingerprint(loggingContext1, configuration1.Startup, configuration1, context1.PathTable, evaluationFilter1, fileContentTable1, "111aaa", null).ExactFingerprint);
}
public async Task TestSerialization()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var symbolTable = new SymbolTable(pathTable.StringTable);
var whitelist = new FileAccessWhitelist(context);
var path1 = AbsolutePath.Create(pathTable, @"\\fakePath\foo.txt");
var path2 = AbsolutePath.Create(pathTable, @"\\fakePath\bar.txt");
var regex1 = new SerializableRegex(@"dir\foo.txt");
var executableEntry1 = new ExecutablePathWhitelistEntry(
path1, regex1, true, "entry1");
var executableEntry2 = new ExecutablePathWhitelistEntry(
path2, new SerializableRegex("bar"), false, "entry2");
whitelist.Add(executableEntry1);
whitelist.Add(executableEntry2);
var symbol1 = FullSymbol.Create(symbolTable, "symbol1");
var valueEntry = new ValuePathFileAccessWhitelistEntry(
symbol1, new SerializableRegex("symbol1"), false, null);
var symbol2 = FullSymbol.Create(symbolTable, "symbol2");
var valueEntry2 = new ValuePathFileAccessWhitelistEntry(
symbol2, new SerializableRegex("symbol2"), false, "entry4");
whitelist.Add(valueEntry);
whitelist.Add(valueEntry2);
XAssert.AreEqual(3, whitelist.UncacheableEntryCount);
XAssert.AreEqual(1, whitelist.CacheableEntryCount);
XAssert.AreEqual("Unnamed", valueEntry.Name);
using (var ms = new MemoryStream())
{
BuildXLWriter writer = new BuildXLWriter(true, ms, true, true);
whitelist.Serialize(writer);
ms.Position = 0;
BuildXLReader reader = new BuildXLReader(true, ms, true);
var deserialized = await FileAccessWhitelist.DeserializeAsync(reader, Task.FromResult <PipExecutionContext>(context));
XAssert.AreEqual(2, deserialized.ExecutablePathEntries.Count);
XAssert.AreEqual(1, deserialized.ExecutablePathEntries[path1].Count);
XAssert.AreEqual(true, deserialized.ExecutablePathEntries[path1][0].AllowsCaching);
XAssert.AreEqual(regex1.ToString(), deserialized.ExecutablePathEntries[path1][0].PathRegex.ToString());
XAssert.AreEqual(executableEntry1.Name, deserialized.ExecutablePathEntries[path1][0].Name);
XAssert.AreEqual(executableEntry2.Name, deserialized.ExecutablePathEntries[path2][0].Name);
XAssert.AreEqual(2, deserialized.ValuePathEntries.Count);
XAssert.AreEqual(1, deserialized.ValuePathEntries[symbol1].Count);
XAssert.AreEqual(false, deserialized.ValuePathEntries[symbol1][0].AllowsCaching);
XAssert.AreEqual(valueEntry.Name, deserialized.ValuePathEntries[symbol1][0].Name);
XAssert.AreEqual(valueEntry2.Name, deserialized.ValuePathEntries[symbol2][0].Name);
XAssert.AreEqual(3, deserialized.UncacheableEntryCount);
XAssert.AreEqual(1, deserialized.CacheableEntryCount);
}
}
public void ReadLevelViolationReportedForUnknownPath()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
AbsolutePath unknown = CreateAbsolutePath(context, X("/X/out/unknown"));
AbsolutePath produced = CreateAbsolutePath(context, ProducedPath);
Process process = graph.AddProcess(produced);
analyzer.AnalyzePipViolations(
process,
new[] { CreateViolation(RequestedAccess.Read, unknown) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.MissingSourceDependency,
FileMonitoringViolationAnalyzer.AccessLevel.Read,
unknown,
process,
null),
"A MissingSourceDependency should have been reported with no suggested value");
analyzer.AssertNoExtraViolationsCollected();
AssertErrorEventLogged(EventId.FileMonitoringError);
}
public void TestDirectoryTranslatorUsedForJunctionsInCloudBuild()
{
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var translations = new[]
{
CreateInputTranslation(pathTable, new string[] { "K", "dbs", "sh", "dtb", "b" }, new string[] { "d", "dbs", "sh", "dtb", "0629_120346" }),
CreateInputTranslation(pathTable, new string[] { "d", "dbs", "sh", "dtb", "0629_120346", "Build" }, new string[] { "d", "dbs", "el", "dtb", "Build" }),
CreateInputTranslation(pathTable, new string[] { "d", "dbs", "sh", "dtb", "0629_120346", "Target" }, new string[] { "d", "dbs", "el", "dtb", "Target" })
};
string error;
XAssert.IsTrue(DirectoryTranslator.ValidateDirectoryTranslation(pathTable, translations, out error));
var translator = new DirectoryTranslator();
translator.AddTranslations(translations, pathTable);
translator.Seal();
AssertEqualTranslatedPath(translator, pathTable, new string[] { "d", "dbs", "el", "dtb", "Build", "x64", "debug", "perl.cmd" }, new string[] { "K", "dbs", "sh", "dtb", "b", "Build", "x64", "debug", "perl.cmd" });
AssertEqualTranslatedPath(translator, pathTable, new string[] { "d", "dbs", "el", "dtb", "Target", "x64", "debug", "perl.cmd" }, new string[] { "K", "dbs", "sh", "dtb", "b", "Target", "x64", "debug", "perl.cmd" });
AssertEqualTranslatedPath(translator, pathTable, new string[] { "d", "dbs", "sh", "dtb", "0629_120346" }, new string[] { "K", "dbs", "sh", "dtb", "b" });
AssertEqualTranslatedPath(translator, new string[] { @"\\?\d", "dbs", "el", "dtb", "Build", "x64", "debug", "perl.cmd" }, new string[] { @"\\?\K", "dbs", "sh", "dtb", "b", "Build", "x64", "debug", "perl.cmd" });
AssertEqualTranslatedPath(translator, new string[] { @"\??\d", "dbs", "el", "dtb", "Build", "x64", "debug", "perl.cmd" }, new string[] { @"\??\K", "dbs", "sh", "dtb", "b", "Build", "x64", "debug", "perl.cmd" });
}
public void UndeclaredReadCycleReportedForPathOrderedAfter()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
AbsolutePath violatorOutput = CreateAbsolutePath(context, JunkPath);
AbsolutePath producerOutput = CreateAbsolutePath(context, ProducedPath);
Process violator = graph.AddProcess(violatorOutput);
Process producer = graph.AddProcess(producerOutput);
analyzer.AnalyzePipViolations(
violator,
new[] { CreateViolation(RequestedAccess.Read, producerOutput) },
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.UndeclaredReadCycle,
FileMonitoringViolationAnalyzer.AccessLevel.Read,
producerOutput,
violator,
producer),
"An UndeclaredReadCycle should have been reported since an earlier pip has an undeclared read of a later pip's output.");
analyzer.AssertNoExtraViolationsCollected();
AssertErrorEventLogged(EventId.FileMonitoringError);
}
/// <summary>
/// Creates a common sandboxed process info for tests in this class.
/// </summary>
private static SandboxedProcessInfo CreateCommonSandboxedProcessInfo(
BuildXLContext context,
string executable,
string arguments,
FileAccessManifest fileAccessManifest,
StringBuilder stdOutBuilder,
StringBuilder stdErrBuilder)
{
return(new SandboxedProcessInfo(
context.PathTable,
new LocalSandboxedFileStorage(),
executable,
disableConHostSharing: true,
loggingContext: CreateLoggingContext(),
fileAccessManifest: fileAccessManifest)
{
PipDescription = executable,
Arguments = arguments,
WorkingDirectory = Environment.CurrentDirectory,
StandardOutputEncoding = Encoding.UTF8,
StandardOutputObserver = stdOutStr => stdOutBuilder.AppendLine(stdOutStr),
StandardErrorEncoding = Encoding.UTF8,
StandardErrorObserver = stdErrStr => stdErrBuilder.AppendLine(stdErrStr),
EnvironmentVariables = BuildParameters.GetFactory().PopulateFromEnvironment(),
Timeout = TimeSpan.FromMinutes(1),
});
}
public void UndeclaredOutput()
{
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
var graph = new QueryablePipDependencyGraph(context);
var analyzer = new TestFileMonitoringViolationAnalyzer(LoggingContext, context, graph);
AbsolutePath declaredOutput = CreateAbsolutePath(context, X("/X/out/produced1"));
AbsolutePath undeclaredOutput = CreateAbsolutePath(context, X("/X/out/produced2"));
Process producer = graph.AddProcess(declaredOutput);
analyzer.AnalyzePipViolations(
producer,
new[]
{
CreateViolation(RequestedAccess.Write, undeclaredOutput),
},
new ReportedFileAccess[0],
exclusiveOpaqueDirectoryContent: null,
sharedOpaqueDirectoryWriteAccesses: null,
allowedUndeclaredReads: null,
absentPathProbesUnderOutputDirectories: null);
analyzer.AssertContainsViolation(
new DependencyViolation(
FileMonitoringViolationAnalyzer.DependencyViolationType.UndeclaredOutput,
FileMonitoringViolationAnalyzer.AccessLevel.Write,
undeclaredOutput,
producer,
null),
"The violator has an undeclared output but it wasn't reported.");
AssertErrorEventLogged(EventId.FileMonitoringError);
}
private void WriteFakeMemosDb(BuildXLContext context, IConfiguration configuration)
{
var memoDbPath = GetMemosDbPath(context, configuration);
Directory.CreateDirectory(Path.GetDirectoryName(memoDbPath));
File.WriteAllText(memoDbPath, Guid.NewGuid().ToString());
}
private static async Task <bool> CreateAndRunPip(
PipProgram program,
string tempDirectory,
string outFile,
IEnumerable <string> restInstructions,
bool is64Bit)
{
Contract.Requires(restInstructions != null);
Contract.Requires(tempDirectory != null);
Contract.Requires(!string.IsNullOrEmpty(outFile));
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
using (var fileAccessListener = new FileAccessListener(Events.Log))
{
fileAccessListener.RegisterEventSource(BuildXL.Processes.ETWLogger.Log);
var fileContentTable = FileContentTable.CreateNew();
var config = ConfigurationHelpers.GetDefaultForTesting(context.PathTable, AbsolutePath.Create(context.PathTable, Path.Combine(tempDirectory, "config.dc")));
config.Sandbox.LogObservedFileAccesses = true;
Pip pip = null;
var instructions = restInstructions as string[] ?? restInstructions.ToArray();
switch (program)
{
case PipProgram.Cmd:
pip = CreateCmdPip(context, tempDirectory, outFile, is64Bit);
break;
case PipProgram.Self:
pip = CreateSelfPip(context, tempDirectory, outFile, instructions, is64Bit);
break;
}
Contract.Assume(pip != null);
PipResult executeResult = await Execute(context, fileContentTable, config, pip);
bool valid = false;
switch (program)
{
case PipProgram.Cmd:
valid = ValidateCmd(fileAccessListener.FileAccesses, outFile, is64Bit);
break;
case PipProgram.Self:
valid = ValidateSelf(
fileAccessListener.FileAccesses,
instructions.Length > 0 ? instructions[0] : string.Empty,
outFile,
is64Bit);
break;
}
return(executeResult.Status == PipResultStatus.Succeeded && valid);
}
}
public PerProcessPipPerformanceInformationStoreTests(ITestOutputHelper output)
: base(output)
{
m_context = BuildXLContext.CreateInstanceForTesting();
m_configuration = ConfigurationHelpers.GetDefaultForTesting(m_context.PathTable, AbsolutePath.Create(m_context.PathTable, Path.Combine(TemporaryDirectory, "config.ds")));
m_perPipPerformanceInformationStore = new PerProcessPipPerformanceInformationStore(m_configuration.Logging.MaxNumPipTelemetryBatches, m_configuration.Logging.AriaIndividualMessageSizeLimitBytes);
}
/// <summary>
/// Initialize the test
/// </summary>
public DirectoryMembershipFingerprinterTests(ITestOutputHelper output) : base(output)
{
m_context = BuildXLContext.CreateInstanceForTesting();
m_pipTable = new PipTable(m_context.PathTable, m_context.SymbolTable, initialBufferSize: 1024, maxDegreeOfParallelism: 1, debug: true);
m_mountInfo = new MountPathExpander(m_context.PathTable);
RegisterEventSource(global::BuildXL.Scheduler.ETWLogger.Log);
}
