public PipQueueTestExecutionEnvironment(BuildXLContext context, IConfiguration configuration, PipTable pipTable, string tempDirectory, ISandboxConnection SandboxConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(configuration != null);
Context = context;
LoggingContext = CreateLoggingContextForTest();
Configuration = configuration;
FileContentTable = FileContentTable.CreateNew();
ContentFingerprinter = new PipContentFingerprinter(
context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
ExtraFingerprintSalts.Default(),
pathExpander: PathExpander);
PipTable = pipTable;
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = IpcFactory.GetProvider();
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
Cache = InMemoryCacheFactory.Create();
LocalDiskContentStore = new LocalDiskContentStore(LoggingContext, context.PathTable, FileContentTable, tracker);
m_sandboxConnectionKext = SandboxConnection;
m_expectedWrittenContent = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_wellKnownFiles = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_producers = new ConcurrentDictionary <FileArtifact, Pip>();
m_filesystemView = new TestPipGraphFilesystemView(Context.PathTable);
var fileSystemView = new FileSystemView(Context.PathTable, m_filesystemView, LocalDiskContentStore);
TempCleaner = new TestMoveDeleteCleaner(tempDirectory);
State = new PipExecutionState(
configuration,
cache: new PipTwoPhaseCache(LoggingContext, Cache, context, PathExpander),
unsafeConfiguration: configuration.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: ContentHashingUtilities.CreateRandom(),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: null,
fileSystemView: fileSystemView,
fileContentManager: new FileContentManager(this, new NullOperationTracker()),
directoryMembershipFinterprinterRuleSet: null);
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
/// <nodoc/>
public SchedulerIntegrationTestBase(ITestOutputHelper output) : base(output)
{
m_testOutputHelper = output;
CaptureAllDiagnosticMessages = false;
// Each event listener that we want to capture events from must be listed here
foreach (var eventSource in BuildXLApp.GeneratedEventSources)
{
RegisterEventSource(eventSource);
}
// Go through the command line configuration handling used by the real app to get the appropriate defaults
ICommandLineConfiguration config;
XAssert.IsTrue(Args.TryParseArguments(new[] { "/c:" + Path.Combine(TemporaryDirectory, "config.dc") }, Context.PathTable, null, out config), "Failed to construct arguments");
Configuration = new CommandLineConfiguration(config);
Cache = InMemoryCacheFactory.Create();
FileContentTable = FileContentTable.CreateNew(LoggingContext);
// Disable defaults that write disk IO but are not critical to correctness or performance
Configuration.Logging.StoreFingerprints = false;
Configuration.Logging.LogExecution = false;
Configuration.Engine.TrackBuildsInUserFolder = false;
Configuration.Engine.CleanTempDirectories = false;
Configuration.Sandbox.EnsureTempDirectoriesExistenceBeforePipExecution = true;
// Skip hash source file pips
// Some error becomes a contract exception when SkipHashSourceFile is enabled.
// Mostly on methods that traverse graph, e.g., DirectedGraph.GetOutgoingEdges();
// BUG 1472567
// Configuration.Schedule.SkipHashSourceFile = true;
// Compute static pip fingerprints for incremental scheduling tests.
Configuration.Schedule.ComputePipStaticFingerprints = true;
// Disable currently enabled unsafe option.
Configuration.Sandbox.UnsafeSandboxConfigurationMutable.IgnoreCreateProcessReport = false;
// Populate file system capabilities.
// Here, for example, we use copy-on-write instead of hardlinks when Unix file system supports copy-on-write.
// Particular tests can override this by setting Configuration.Engine.UseHardlinks.
BuildXLEngine.PopulateFileSystemCapabilities(Configuration, Configuration, Context.PathTable, LoggingContext);
// Reset pip graph builder to use the populated configuration.
ResetPipGraphBuilder();
}
public async Task TruncatedTableResultsInRecoverableException()
{
WriteTestFiles();
var originalTable = FileContentTable.CreateNew();
RecordContentHash(originalTable, m_testFileA, s_hashA);
RecordContentHash(originalTable, m_testFileB, s_hashB);
await SaveTable(originalTable);
TruncateTable();
var fileContentTable = await LoadTable();
XAssert.AreEqual(null, fileContentTable, "Table shouldn't have loaded due to missing data");
}
public void RecycledFileContentTableKeepsInformation()
{
WriteTestFiles();
var originalTable = FileContentTable.CreateNew(LoggingContext);
RecordContentHash(originalTable, m_testFileA, s_hashA);
RecordContentHash(originalTable, m_testFileB, s_hashB);
var newTable = FileContentTable.CreateFromTable(originalTable, LoggingContext);
ExpectHashKnown(newTable, m_testFileA, s_hashA);
ExpectHashKnown(newTable, m_testFileB, s_hashB);
VerifyTable(newTable);
}
public async Task CanRoundTripMultipleEntries()
{
WriteTestFiles();
var originalTable = FileContentTable.CreateNew();
RecordContentHash(originalTable, m_testFileA, s_hashA);
RecordContentHash(originalTable, m_testFileB, s_hashB);
FileContentTable loadedTable = await SaveAndReloadTable(originalTable);
ExpectHashKnown(loadedTable, m_testFileA, s_hashA);
ExpectHashKnown(loadedTable, m_testFileB, s_hashB);
VerifyTable(loadedTable);
}
public void GenerateHashWithDifferentEvaluationFilters()
{
WriteFile("config.ds", "SampleConfig");
var context1 = BuildXLContext.CreateInstanceForTesting();
var configuration1 = ConfigurationHelpers.GetDefaultForTesting(context1.PathTable, AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, "config.ds")));
var evaluationFilter1 = new EvaluationFilter(
context1.SymbolTable,
context1.PathTable,
new FullSymbol[0],
new[]
{
AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"testFile1.txt")),
},
CollectionUtilities.EmptyArray <StringId>());
var evaluationFilter2 = new EvaluationFilter(
context1.SymbolTable,
context1.PathTable,
new FullSymbol[0],
new[]
{
AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"testFile2.txt")),
},
CollectionUtilities.EmptyArray <StringId>());
configuration1.Layout.ObjectDirectory = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"ObjectDirectory1"));
configuration1.Layout.TempDirectory = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"TempDirectory1"));
configuration1.Layout.SourceDirectory = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"SourceDirectory1"));
configuration1.Logging.SubstTarget = AbsolutePath.Create(context1.PathTable, Path.Combine(TemporaryDirectory, $"SubstTarget1"));
configuration1.Engine.CompressGraphFiles = false;
configuration1.Schedule.SkipHashSourceFile = false;
configuration1.Schedule.ComputePipStaticFingerprints = false;
var loggingContext1 = CreateLoggingContextForTest();
var fileContentTable1 = FileContentTable.CreateNew(loggingContext1);
var oldFingerprint = GraphFingerprinter.TryComputeFingerprint(loggingContext1, configuration1.Startup, configuration1, context1.PathTable, evaluationFilter1, fileContentTable1, "111aaa", null).ExactFingerprint;
var newFingerprint1 = GraphFingerprinter.TryComputeFingerprint(loggingContext1, configuration1.Startup, configuration1, context1.PathTable, evaluationFilter2, fileContentTable1, "111aaa", null).ExactFingerprint;
var comparison = oldFingerprint.CompareFingerprint(newFingerprint1);
Assert.Equal(GraphCacheMissReason.EvaluationFilterChanged, comparison);
}
public async Task ChangedContentInvalidatesEntryAfterRoundtrip()
{
WriteTestFiles();
var originalTable = FileContentTable.CreateNew();
RecordContentHash(originalTable, m_testFileA, s_hashA);
ExpectHashKnown(originalTable, m_testFileA, s_hashA);
await SaveTable(originalTable);
ModifyContents(m_testFileA);
FileContentTable loadedTable = await LoadTable();
ExpectHashUnknown(loadedTable, m_testFileA);
VerifyTable(loadedTable);
}
public void RecyclingFileContentTableIsTheSameAsSavingAndLoadingFromFile()
{
WriteTestFiles();
var table = FileContentTable.CreateNew(LoggingContext, entryTimeToLive: 2);
RecordContentHash(table, m_testFileA, s_hashA); // TTL(A) = 2
table = FileContentTable.CreateFromTable(table, LoggingContext); // TTL(A) = 1
table = FileContentTable.CreateFromTable(table, LoggingContext); // TTL(A) = 0
RecordContentHash(table, m_testFileB, s_hashB); // TTL(B) = 2
table = FileContentTable.CreateFromTable(table, LoggingContext);
ExpectHashUnknown(table, m_testFileA); // A Evicted
ExpectHashKnown(table, m_testFileB, s_hashB); // TTL(B) = 1
VerifyTable(table);
}
public async Task UnusedEntriesAreEvictedWithMinimumTimeToLive()
{
WriteTestFiles();
var originalTable = FileContentTable.CreateNew(entryTimeToLive: 1);
RecordContentHash(originalTable, m_testFileA, s_hashA);
RecordContentHash(originalTable, m_testFileB, s_hashB);
FileContentTable loadedTable = await SaveAndReloadTable(originalTable);
ExpectHashKnown(loadedTable, m_testFileB, s_hashB);
FileContentTable laterLoadedTable = await SaveAndReloadTable(loadedTable);
ExpectHashKnown(laterLoadedTable, m_testFileB, s_hashB);
ExpectHashUnknown(laterLoadedTable, m_testFileA);
VerifyTable(laterLoadedTable);
}
public async Task GetAndRecordContentHashAsyncWithMatch()
{
ContentHash fakeHash = ContentHashingUtilities.CreateRandom();
var fileContentTable = FileContentTable.CreateNew();
WriteFile(FileA, "Some string");
using (FileStream fs = File.OpenRead(GetFullPath(FileA)))
{
// Note that this content hash is clearly wrong, but also not all zeroes.
fileContentTable.RecordContentHash(fs, fakeHash);
}
FileContentTableExtensions.VersionedFileIdentityAndContentInfoWithOrigin result =
await fileContentTable.GetAndRecordContentHashAsync(GetFullPath(FileA));
XAssert.AreEqual(FileContentTableExtensions.ContentHashOrigin.Cached, result.Origin);
XAssert.AreEqual(fakeHash, result.VersionedFileIdentityAndContentInfo.FileContentInfo.Hash);
}
public async Task TruncatedTableResultsCanBeDiscardedWithWarning()
{
WriteTestFiles();
var originalTable = FileContentTable.CreateNew();
RecordContentHash(originalTable, m_testFileA, s_hashA);
RecordContentHash(originalTable, m_testFileB, s_hashB);
await SaveTable(originalTable);
TruncateTable();
FileContentTable loadedTable = await LoadOrCreateTable();
ExpectHashUnknown(loadedTable, m_testFileA);
ExpectHashUnknown(loadedTable, m_testFileB);
AssertInformationalEventLogged(EventId.StorageLoadFileContentTable);
VerifyTable(loadedTable);
}
public async Task WriteBytesIfContentMismatchedAsyncWithMismatchedDestination()
{
const string TargetContent = "Target!!!!!!!!!";
var fileContentTable = FileContentTable.CreateNew();
WriteFile(FileB, "Nope");
FileContentTableExtensions.VersionedFileIdentityAndContentInfoWithOrigin targetInfo =
await fileContentTable.GetAndRecordContentHashAsync(GetFullPath(FileB));
XAssert.AreEqual(FileContentTableExtensions.ContentHashOrigin.NewlyHashed, targetInfo.Origin);
await
VerifyWriteBytesIfContentMismatchedAsync(
fileContentTable,
FileB,
TargetContent,
expectWrite : true,
originalDestinationInfo : targetInfo.VersionedFileIdentityAndContentInfo);
}
public async Task GetAndRecordContentHashAsyncWithMismatch()
{
var fileContentTable = FileContentTable.CreateNew();
using (FileStream fs = File.Open(GetFullPath(FileA), FileMode.CreateNew, FileAccess.Write))
{
// Register an empty file.
fileContentTable.RecordContentHash(fs, ContentHashingUtilities.EmptyHash);
byte[] content = Encoding.UTF8.GetBytes("Some new content");
await fs.WriteAsync(content, 0, content.Length);
}
FileContentTableExtensions.VersionedFileIdentityAndContentInfoWithOrigin result =
await fileContentTable.GetAndRecordContentHashAsync(GetFullPath(FileA));
XAssert.AreEqual(FileContentTableExtensions.ContentHashOrigin.NewlyHashed, result.Origin);
XAssert.AreEqual(
await ContentHashingUtilities.HashFileAsync(GetFullPath(FileA)),
result.VersionedFileIdentityAndContentInfo.FileContentInfo.Hash);
}
public async Task CopyIfContentMismatchedAsyncWithAbsentDestination()
{
const string TargetContent = "Target!!!!!!!!!";
var fileContentTable = FileContentTable.CreateNew();
WriteFile(FileA, TargetContent);
FileContentTableExtensions.VersionedFileIdentityAndContentInfoWithOrigin sourceInfo =
await fileContentTable.GetAndRecordContentHashAsync(GetFullPath(FileA));
XAssert.AreEqual(FileContentTableExtensions.ContentHashOrigin.NewlyHashed, sourceInfo.Origin);
await
VerifyCopyIfContentMismatchedAsync(
fileContentTable,
FileA,
FileB,
sourceInfo.VersionedFileIdentityAndContentInfo.FileContentInfo,
expectedCopy : true);
}
public void StrictModeFlushesDirtyMemoryMappedPages()
{
var table = FileContentTable.CreateNew();
string testFileAExpandedPath = m_testFileA.ToString(m_pathTable);
// Writing a file via a memory mapping leaves 'dirty' pages in cache that get lazily flushed to FS.
// We want to verify that passing strict: true flushes those pages, so that the recorded USN is stable (not invalidated due to lazy flushes).
using (FileStream file = FileUtilities.CreateFileStream(testFileAExpandedPath, FileMode.Create, FileAccess.ReadWrite, FileShare.Delete | FileShare.Read))
{
file.SetLength(1024 * 1024);
using (var memoryMappedFile = MemoryMappedFile.CreateFromFile(
file,
mapName: null,
capacity: 0,
access: MemoryMappedFileAccess.ReadWrite,
#if !DISABLE_FEATURE_MEMORYMAP_SECURITY
memoryMappedFileSecurity: null,
#endif
inheritability: HandleInheritability.None,
leaveOpen: false))
{
using (var accessor = memoryMappedFile.CreateViewAccessor())
{
for (int i = 0; i < 1024 * 1024; i += 4)
{
accessor.Write(i, (int)0xAB);
}
}
}
RecordContentHash(table, m_testFileA, s_hashA, strict: true);
}
for (int i = 0; i < 10; i++)
{
Thread.Sleep(10);
ExpectHashKnown(table, m_testFileA, s_hashA);
}
}
/// <summary>
/// Creates a file content table.
/// </summary>
protected FileContentTable CreateFileContentTable()
{
return(FileContentTable.CreateNew(LoggingContext));
}
private async Task ProcessWindowsCallHelper(
string functionName,
SandboxConfiguration config = null,
IEnumerable <string> extraDependencies = null,
IEnumerable <string> extraOutputs = null,
int callCount = 1,
string commandPrefix = "",
bool readsAndWritesDirectories = false,
bool untrackedOutputs = false,
string[] expectedWarningStrings = null,
string[] expectedErrorStrings = null)
{
if (config == null)
{
config = new SandboxConfiguration {
FileAccessIgnoreCodeCoverage = true, FailUnexpectedFileAccesses = true
};
}
var context = BuildXLContext.CreateInstanceForTesting();
var pathTable = context.PathTable;
var fileContentTable = FileContentTable.CreateNew(LoggingContext);
// have to force the config for truncation
config.OutputReportingMode = OutputReportingMode.FullOutputOnWarningOrError;
bool expectSuccess = expectedErrorStrings == null && expectedWarningStrings == null;
using (var tempFiles = new TempFileStorage(canGetFileNames: true, rootPath: TemporaryDirectory))
{
string currentCodeFolder = Path.GetDirectoryName(AssemblyHelper.GetAssemblyLocation(Assembly.GetExecutingAssembly()));
Contract.Assume(currentCodeFolder != null);
string executable = Path.Combine(currentCodeFolder, DetourTestFolder, "DetoursTests.exe");
string workingDirectory = tempFiles.GetUniqueDirectory();
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
XAssert.IsTrue(File.Exists(executable), "Could not find the test file: " + executable);
FileArtifact executableFileArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
var extraUntrackedScopes = new List <AbsolutePath>();
var dependencies = new List <FileArtifact> {
executableFileArtifact
};
if (extraDependencies != null)
{
foreach (string file in extraDependencies)
{
string filePath = Path.Combine(workingDirectory, file);
AbsolutePath path = AbsolutePath.Create(pathTable, filePath);
if (readsAndWritesDirectories)
{
Directory.CreateDirectory(filePath);
// We don't support directories as inputs in BuildXL yet.
extraUntrackedScopes.Add(path);
}
else
{
File.WriteAllText(filePath, "Definitely a file");
FileArtifact fileArtifact = FileArtifact.CreateSourceFile(path);
dependencies.Add(fileArtifact);
}
}
}
var outputs = new List <FileArtifactWithAttributes>();
if (extraOutputs != null)
{
foreach (string file in extraOutputs)
{
string filePath = Path.Combine(workingDirectory, file);
AbsolutePath path = AbsolutePath.Create(pathTable, filePath);
if (readsAndWritesDirectories)
{
// We don't support directory outputs in BuildXL at the moment, so e.g. deleting a directory needs to be untracked.
extraUntrackedScopes.Add(path);
}
else if (untrackedOutputs)
{
extraUntrackedScopes.Add(path);
}
else
{
FileArtifact fileArtifact = FileArtifact.CreateSourceFile(path).CreateNextWrittenVersion();
outputs.Add(fileArtifact.WithAttributes());
}
}
}
var tempDirectory = tempFiles.GetUniqueDirectory();
var environmentVariables = new List <EnvironmentVariable>();
var environmentValue = new PipDataBuilder(pathTable.StringTable);
var tempPath = AbsolutePath.Create(pathTable, tempDirectory);
environmentValue.Add(tempPath);
environmentVariables.Add(new EnvironmentVariable(StringId.Create(pathTable.StringTable, "TMP"), environmentValue.ToPipData(" ", PipDataFragmentEscaping.NoEscaping)));
environmentVariables.Add(new EnvironmentVariable(StringId.Create(pathTable.StringTable, "TEMP"), environmentValue.ToPipData(" ", PipDataFragmentEscaping.NoEscaping)));
var untrackedPaths = CmdHelper.GetCmdDependencies(pathTable);
var untrackedScopes = extraUntrackedScopes.Concat(CmdHelper.GetCmdDependencyScopes(pathTable).Concat(new[] { tempPath })).Distinct();
var pip = new Process(
executableFileArtifact,
workingDirectoryAbsolutePath,
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.NoEscaping, commandPrefix + functionName + "Logging"),
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .From(environmentVariables),
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .From(dependencies),
ReadOnlyArray <FileArtifactWithAttributes> .From(outputs),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(untrackedPaths),
ReadOnlyArray <AbsolutePath> .From(untrackedScopes),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
if (expectSuccess)
{
await AssertProcessSucceedsAsync(
context,
config,
pip);
}
else
{
await AssertProcessCompletesWithStatusAsync(
SandboxedProcessPipExecutionStatus.ExecutionFailed,
context,
config,
pip,
null);
}
}
int expectedErrorCount = 0;
int expectedWarningCount = 0;
IEnumerable <string> requiredLogMessageSubstrings = new string[] { };
if (expectedErrorStrings != null)
{
expectedErrorCount = expectedErrorStrings.Count();
requiredLogMessageSubstrings = requiredLogMessageSubstrings.Concat(expectedErrorStrings);
}
if (expectedWarningStrings != null)
{
expectedWarningCount = expectedWarningStrings.Count();
requiredLogMessageSubstrings = requiredLogMessageSubstrings.Concat(expectedWarningStrings);
}
SetExpectedFailures(expectedErrorCount, expectedWarningCount, requiredLogMessageSubstrings.ToArray());
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxedKextConnection = sandboxedKextConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create(context);
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
/// <summary>
/// Creates a file content table.
/// </summary>
protected FileContentTable CreateFileContentTable()
{
Contract.Ensures(Contract.Result <FileContentTable>() != null);
return(FileContentTable.CreateNew(LoggingContext));
}
private static async Task <bool> CreateAndRunPip(
PipProgram program,
string tempDirectory,
string outFile,
IEnumerable <string> restInstructions,
bool is64Bit)
{
Contract.Requires(restInstructions != null);
Contract.Requires(tempDirectory != null);
Contract.Requires(!string.IsNullOrEmpty(outFile));
BuildXLContext context = BuildXLContext.CreateInstanceForTesting();
using (var fileAccessListener = new FileAccessListener(Events.Log))
{
fileAccessListener.RegisterEventSource(BuildXL.Processes.ETWLogger.Log);
var loggingContext = BuildXLTestBase.CreateLoggingContextForTest();
var fileContentTable = FileContentTable.CreateNew(loggingContext);
var config = ConfigurationHelpers.GetDefaultForTesting(context.PathTable, AbsolutePath.Create(context.PathTable, Path.Combine(tempDirectory, "config.dc")));
config.Sandbox.LogObservedFileAccesses = true;
Pip pip = null;
var instructions = restInstructions as string[] ?? restInstructions.ToArray();
switch (program)
{
case PipProgram.Cmd:
pip = CreateCmdPip(context, tempDirectory, outFile, is64Bit);
break;
case PipProgram.Self:
pip = CreateSelfPip(context, tempDirectory, outFile, instructions, is64Bit);
break;
}
Contract.Assume(pip != null);
var isSubstUsed = FileUtilities.TryGetSubstSourceAndTarget(tempDirectory, out var substSource, out var substTarget, out var errorMessage);
XAssert.IsFalse(!isSubstUsed && errorMessage != null, errorMessage);
PipResult executeResult = await Execute(
context,
fileContentTable,
config,
pip,
isSubstUsed
?(substSource, substTarget)
: default((string, string)?));
bool valid = false;
switch (program)
{
case PipProgram.Cmd:
valid = ValidateCmd(fileAccessListener.FileAccesses, outFile, is64Bit);
break;
case PipProgram.Self:
valid = ValidateSelf(
fileAccessListener.FileAccesses,
instructions.Length > 0 ? instructions[0] : string.Empty,
outFile,
is64Bit);
break;
}
return(executeResult.Status == PipResultStatus.Succeeded && valid);
}
}
public async Task ParallelRecordForHardlinksRemembersHash()
{
if (!FileUtilities.IsPreciseFileVersionSupportedByEnlistmentVolume)
{
// TODO: Currently fails on OS that does not support precise file version.
return;
}
var table = FileContentTable.CreateNew(LoggingContext);
const int ThreadCount = 16;
const int IterationCount = 20;
const string OriginalFile = "Original";
WriteFile(OriginalFile, "Data");
CreateHardLinkStatus testLinkStatus = FileUtilities.TryCreateHardLink(GetFullPath("TestLink"), GetFullPath(OriginalFile));
if (testLinkStatus == CreateHardLinkStatus.FailedSinceNotSupportedByFilesystem)
{
return;
}
XAssert.AreEqual(CreateHardLinkStatus.Success, testLinkStatus);
var linkTasks = new Task[ThreadCount];
var barrier = new Barrier(ThreadCount);
for (int j = 0; j < IterationCount; j++)
{
for (int i = 0; i < ThreadCount; i++)
{
int threadId = i;
linkTasks[threadId] = Task.Run(
() =>
{
string relativePath = "Link-" + threadId;
AbsolutePath path = GetFullPath(m_pathTable, relativePath);
barrier.SignalAndWait();
FileUtilities.DeleteFile(GetFullPath(relativePath));
CreateHardLinkStatus linkStatus = FileUtilities.TryCreateHardLink(GetFullPath(relativePath), GetFullPath(OriginalFile));
#if PLATFORM_OSX
// Catalina seems to have issues when doing mutli-threaded delete / link operations, let's retry several times..
if (linkStatus != CreateHardLinkStatus.Success)
{
var count = 0;
while (count < ThreadCount)
{
FileUtilities.DeleteFile(GetFullPath(relativePath));
linkStatus = FileUtilities.TryCreateHardLink(GetFullPath(relativePath), GetFullPath(OriginalFile));
count++;
}
}
#endif
XAssert.AreEqual(CreateHardLinkStatus.Success, linkStatus);
RecordContentHash(table, path, s_hashA);
});
}
await Task.WhenAll(linkTasks);
ExpectHashKnown(table, GetFullPath(m_pathTable, OriginalFile), s_hashA);
}
}
