public void BreakawayProcessesCanOutliveThePip() { var pidFile = CreateOutputFileArtifact(ObjectRoot, prefix: $"{nameof(BreakawayProcessesCanOutliveThePip)}.pid"); var builder = CreatePipBuilder(new Operation[] { Operation.SpawnAndWritePidFile(Context.PathTable, waitToFinish: false, pidFile: pidFile, doNotInfer: false, Operation.Block()) }); // Configure the test process itself to escape the sandbox builder.ChildProcessesToBreakawayFromSandbox = ReadOnlyArray <PathAtom> .FromWithoutCopy(new[] { PathAtom.Create(Context.StringTable, TestProcessToolName) }); var pip = SchedulePipBuilder(builder); RunScheduler().AssertSuccess(); var pidFilePath = ToString(pidFile); XAssert.FileExists(pidFilePath); var pidFileContent = File.ReadAllText(pidFilePath); XAssert.IsTrue(int.TryParse(pidFileContent, out var pid), $"Cannot convert pid file content '{pidFileContent}' to integer"); var proc = TryGetProcessById(pid); XAssert.IsNotNull(proc, $"Could not find the process (PID:{pid}) that was supposed to break away"); proc.Kill(); }
public void DeleteFilesCancellationDoesNotCrash() { const int FileDeletionsAllowed = 2; var testHook = new DirectoryScrubber.TestHooks { OnDeletion = new Action <string, int>((path, numDeletedSoFar) => { if (numDeletedSoFar > FileDeletionsAllowed) { m_cancellationTokenSource.Cancel(); } }) }; string rootDir = Path.Combine(TemporaryDirectory, nameof(DeleteFilesCanDeleteFile)); List <string> files = new List <string>(); for (var i = 0; i < FileDeletionsAllowed * 3; i++) { string fullFilePath = WriteFile(Path.Combine(rootDir, $"out{i}.txt")); XAssert.FileExists(fullFilePath); files.Add(fullFilePath); } XAssert.IsFalse(m_cancellationTokenSource.IsCancellationRequested); var numDeleted = Scrubber.DeleteFiles(files.ToArray(), testHook: testHook); XAssert.IsTrue(m_cancellationTokenSource.IsCancellationRequested); XAssert.Equals(FileDeletionsAllowed, numDeleted); }
private void InvalidateSidebandFile(string sidebandFile, SidebandIntegrityCheckFailReason kind) { XAssert.FileExists(sidebandFile, "Sideband file for pipA not found."); switch (kind) { case SidebandIntegrityCheckFailReason.FileNotFound: AssertDeleteFile(sidebandFile, "Could not delete sideband file for pipA."); break; case SidebandIntegrityCheckFailReason.ChecksumMismatch: File.WriteAllText(path: sidebandFile, contents: "bogus sideband file"); break; case SidebandIntegrityCheckFailReason.MetadataMismatch: SidebandMetadata alteredMetadata; // read the header and the metadata from the original using (var reader = new SidebandReader(sidebandFile)) { XAssert.IsTrue(reader.ReadHeader(ignoreChecksum: false)); var originalMetadata = reader.ReadMetadata(); alteredMetadata = new SidebandMetadata(originalMetadata.PipSemiStableHash + 1, originalMetadata.StaticPipFingerprint); } // overwrite the original with a different metadata file where PiPSemiStableHash is different than in the original using (var writer = new SidebandWriter(alteredMetadata, sidebandFile, null)) { writer.EnsureHeaderWritten(); } break; default: XAssert.Fail($"Unknown kind: {kind}"); break; } }
public void TestSidebandFileProducedUponCacheHits() { // Setup: PipA => sharedOpaqueDir var sharedOpaqueDir = Path.Combine(ObjectRoot, $"sod-{nameof(TestSidebandFileProducedUponCacheHits)}"); var outputInSharedOpaque = CreateOutputFileArtifact(sharedOpaqueDir); var pipA = CreateAndScheduleSharedOpaqueProducer(sharedOpaqueDir, filesToProduceDynamically: outputInSharedOpaque); // first build: cache miss, deletions not postponed, writes are journaled var result = RunScheduler().AssertCacheMiss(pipA.Process.PipId); AssertWritesJournaled(result, pipA, outputInSharedOpaque); AssertSharedOpaqueOutputDeletionNotPostponed(); XAssert.FileExists(ToString(outputInSharedOpaque)); // second build: cache hit, deletions be postponed, writes journaled result = RunScheduler().AssertCacheHit(pipA.Process.PipId); AssertWritesJournaled(result, pipA, outputInSharedOpaque); AssertSharedOpaqueOutputDeletionPostponed(numLazilyDeleted: 0); XAssert.FileExists(ToString(outputInSharedOpaque)); // delete sideband file and build again: // => cache hit // => deletions not postponed (because sideband is missing) // => writes are still journaled even on cache hits AssertDeleteFile(GetSidebandFile(result, pipA.Process)); result = RunScheduler().AssertCacheHit(pipA.Process.PipId); AssertWritesJournaled(result, pipA, outputInSharedOpaque); AssertSharedOpaqueOutputDeletionNotPostponed(); XAssert.FileExists(ToString(outputInSharedOpaque)); }
public void ChangeFile(RepoConfig repoCfg) { using var helper = Clone(repoCfg); // track an existing file var file = helper.GetPath(@"src\files\changingFile.txt"); XAssert.FileExists(file); var oldContent = File.ReadAllText(file); helper.TrackPath(file); // switch to a new branch where that file is modified using var reseter = helper.GitCheckout("changingFile1"); // snap USN entries before doing any checks (because they can modify the journal too) helper.SnapCheckPoint(); // assert that the file exists and has different content in the new branch XAssert.FileExists(file); XAssert.AreNotEqual(oldContent, File.ReadAllText(file)); // assert that the journal recorded a change // (it doesn't matter to us whether that change is 'Delete' or 'Change') helper.AssertDeleteOrChangeFile(file); }
public void ChangeFileWithMaterialization(RepoConfig repoCfg) { using var helper = Clone(repoCfg); // track an existing file var file = helper.GetPath(@"src\files\changingFile.txt"); var result = helper.TrackPath(file); XAssert.AreEqual(PathExistence.ExistsAsFile, result.Existence); var oldContent = File.ReadAllText(file); // switch to a new branch where that file is modified using var reseter = helper.GitCheckout("changingFile1"); // materialize the file (by probing it) before snapping USN entries helper.AssertFileOnDisk(file); helper.SnapCheckPoint(); // assert that the file exists and has different content in the new branch XAssert.FileExists(file); XAssert.AreNotEqual(oldContent, File.ReadAllText(file)); // assert that the journal recorded a change // (it doesn't matter to us whether that change is 'Delete' or 'Change') helper.AssertDeleteOrChangeFile(file); }
public void TestAbsentFileAndDir(RepoConfig repoCfg) { using var helper = Clone(repoCfg); // track a file that doesn't exist var dir = helper.GetPath(@"src\files"); var absentDir = helper.GetPath(@"src\files\newSubfolder"); var absentFile = helper.GetPath(@"src\files\newSubfolder\newSubfolderFile.txt"); var dirResult = helper.TrackDir(dir); var absentDirResult = helper.TrackDir(absentDir); var absentFileResult = helper.TrackPath(absentFile); XAssert.AreEqual(PathExistence.ExistsAsDirectory, dirResult.Existence); XAssert.AreEqual(PathExistence.Nonexistent, absentDirResult.Existence); XAssert.AreEqual(PathExistence.Nonexistent, absentFileResult.Existence); // switch to a new branch where that file does exist using var reseter = helper.GitCheckout("newFileInNewSubfolder"); // immediately snap changes helper.SnapCheckPoint(); XAssert.FileExists(absentFile); XAssert.DirectoryExists(absentDir); // assert directory membership changes helper.AssertAnyChange(dir, PathChanges.MembershipChanged); helper.AssertAnyChange(absentDir, PathChanges.NewlyPresentAsDirectory); helper.AssertAnyChange(absentFile, PathChanges.NewlyPresentAsFile); }
public void TestStashingAndUnstashing(RepoConfig repoCfg) { using var helper = Clone(repoCfg); var file = helper.GetPath(@"src\files\changingFile.txt"); XAssert.FileExists(file); TestOutput.WriteLine("Modifying file: " + file); File.AppendAllText(file, "hi"); var modifiedContent = File.ReadAllText(file); var result = helper.TrackPath(file); XAssert.AreEqual(PathExistence.ExistsAsFile, result.Existence); // stash changes, assert that 'Change' or 'Delete' USN entry was recorded helper.Git("stash"); helper.SnapCheckPoint(); XAssert.AreNotEqual(modifiedContent, File.ReadAllText(file)); helper.AssertDeleteOrChangeFile(file); // unfortunately, GVFS projection seems to change even though it probably shoudn't // helper.AssertNoChange(helper.GetGvfsProjectionFilePath()); // must re-track the same path because now it could be a different physical file helper.TrackPath(file); // unstash changes, assert that 'Change' or 'Delete' USN entry was recorded and that GVFS projection hasn't changed helper.Git("stash pop"); helper.SnapCheckPoint(); XAssert.AreEqual(modifiedContent, File.ReadAllText(file)); helper.AssertDeleteOrChangeFile(file); // unfortunately, GVFS projection seems to change even though it probably shouldn't // helper.AssertNoChange(helper.GetGvfsProjectionFilePath()); }
public void TestFolderBecomesFile(RepoConfig repoCfg) { using var helper = Clone(repoCfg); // track two directories var dir = helper.GetPath(@"src\files"); var subdir = helper.GetPath(@"src\files\subfolder"); var dirResult = helper.TrackDir(dir); var subdirResult = helper.TrackDir(subdir); XAssert.AreEqual(PathExistence.ExistsAsDirectory, dirResult.Existence); XAssert.AreEqual(PathExistence.ExistsAsDirectory, subdirResult.Existence); // switch to a new branch where the subdirectory is now a file using var reseter = helper.GitCheckout("folderBecomesFile"); // immediately snap changes helper.SnapCheckPoint(); XAssert.FileExists(subdir); // assert directory membership changes helper.AssertAnyChange(dir, PathChanges.MembershipChanged); helper.AssertAnyChange(subdir, PathChanges.Removed); }
public void DeleteFilesWithPreExistingCancellationDoesNotCrash() { string rootDir = Path.Combine(TemporaryDirectory, nameof(DeleteFilesCanDeleteFile)); string fullFilePath = WriteFile(Path.Combine(rootDir, "out.txt")); XAssert.FileExists(fullFilePath); m_cancellationTokenSource.Cancel(); var numDeleted = Scrubber.DeleteFiles(new[] { fullFilePath }); XAssert.FileExists(fullFilePath); XAssert.AreEqual(0, numDeleted); }
public void TestBasicCaching() { // Setup: PipA => sharedOpaqueDir => PipB var sharedOpaqueDir = Path.Combine(ObjectRoot, $"sod-{nameof(TestBasicCaching)}"); var outputInSharedOpaque = CreateOutputFileArtifact(sharedOpaqueDir); var source = CreateSourceFile(); var pipA = CreateAndScheduleSharedOpaqueProducer( sharedOpaqueDir, fileToProduceStatically: FileArtifact.Invalid, sourceFileToRead: source, filesToProduceDynamically: outputInSharedOpaque); var builderB = CreatePipBuilder(new Operation[] { Operation.ReadFile(outputInSharedOpaque, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()) }); builderB.AddInputDirectory(pipA.ProcessOutputs.GetOpaqueDirectory(ToPath(sharedOpaqueDir))); var pipB = SchedulePipBuilder(builderB); // B should be able to consume the file in the opaque directory. var result = RunScheduler().AssertCacheMiss(pipA.Process.PipId, pipB.Process.PipId); AssertWritesJournaled(result, pipA, outputInSharedOpaque); AssertSharedOpaqueOutputDeletionNotPostponed(); XAssert.FileExists(ToString(outputInSharedOpaque)); // Second build should have both cache hits; deletions should be postponed RunScheduler().AssertCacheHit(pipA.Process.PipId, pipB.Process.PipId); AssertSharedOpaqueOutputDeletionPostponed(numLazilyDeleted: 0); XAssert.FileExists(ToString(outputInSharedOpaque)); // Make sure we can replay the file in the opaque directory; still cache hits and deletions should be postponed File.Delete(ToString(outputInSharedOpaque)); RunScheduler().AssertCacheHit(pipA.Process.PipId, pipB.Process.PipId); AssertSharedOpaqueOutputDeletionPostponed(numLazilyDeleted: 0); XAssert.FileExists(ToString(outputInSharedOpaque)); // Modify the input and make sure both are rerun, deletions should still be postponed File.WriteAllText(ToString(source), "New content"); RunScheduler().AssertCacheMiss(pipA.Process.PipId, pipB.Process.PipId); AssertSharedOpaqueOutputDeletionPostponed(numLazilyDeleted: 1); }
public void DirectorySymlinksUnderSharedOpaquesArePreservedIfNonEmpty() { string rootDir = Path.Combine(TemporaryDirectory, nameof(DirectorySymlinksUnderSharedOpaquesArePreservedIfNonEmpty)); string fullTargetDirPath = Path.Combine(rootDir, "target-dir"); Directory.CreateDirectory(fullTargetDirPath); XAssert.IsTrue(Directory.Exists(fullTargetDirPath)); var fileUnderTarget = Path.Combine(fullTargetDirPath, "file.txt"); File.WriteAllText(fileUnderTarget, "content"); string fullSymlinkPath = WriteSymlink(Path.Combine(rootDir, "directory symlink"), fullTargetDirPath, isTargetFile: false); XAssert.IsTrue(FileUtilities.FileExistsNoFollow(fullSymlinkPath)); if (OperatingSystemHelper.IsMacOS) { SharedOpaqueOutputHelper.EnforceFileIsSharedOpaqueOutput(fullSymlinkPath); XAssert.IsTrue(SharedOpaqueOutputHelper.IsSharedOpaqueOutput(fullSymlinkPath)); } // This is somewhat subtle. On Windows, IsSharedOpaqueOutput will say yes for any directory, including symlink directories. This means they won't be // considered part of the build and therefore should be traversed. // So if the symlink is traversed, fileUnderTarget will be found, which is not a shared opaque output. So the file won't be deleted. And // so nor the symlink directory. If the symlink directory wasn't traversed, then it would be deleted. Scrubber.RemoveExtraneousFilesAndDirectories( isPathInBuild: path => !SharedOpaqueOutputHelper.IsSharedOpaqueOutput(path), pathsToScrub: new[] { rootDir }, blockedPaths: CollectionUtilities.EmptyArray <string>(), nonDeletableRootDirectories: CollectionUtilities.EmptyArray <string>()); XAssert.FileExists(fileUnderTarget); // On Mac: // - any symlink is a file, any file under shared opaque dir gets scrubber ==> fullSymlinkPath should be scrubbed // // On Windows: // - directories under shared opaques are always removed unless they have files underneath that shouldn't be // removed. This test verifies this behavior also applies to symlink directories XAssert.AreEqual(!OperatingSystemHelper.IsMacOS, Directory.Exists(fullSymlinkPath)); }
public void TestSidebandFileIsAlwaysProducedForPipsWithSharedOpaqueDirectoryOutputs() { // Setup: PipA => sharedOpaqueDir => PipB var sharedOpaqueDir = Path.Combine(ObjectRoot, $"sod-{nameof(TestSidebandFileIsAlwaysProducedForPipsWithSharedOpaqueDirectoryOutputs)}"); var pipA = CreateAndScheduleSharedOpaqueProducer( sharedOpaqueDir, fileToProduceStatically: FileArtifact.Invalid, sourceFileToRead: CreateSourceFile(), filesToProduceDynamically: new FileArtifact[0]); var pipB = CreateAndSchedulePipBuilder(new Operation[] { Operation.WriteFile(CreateOutputFileArtifact()) }); var result = RunScheduler().AssertCacheMiss(pipA.Process.PipId, pipB.Process.PipId); XAssert.FileExists(GetSidebandFile(result, pipA.Process), "Sideband file must be produced for all pips with SOD outputs, even when they don't write a single file into SOD"); XAssert.FileDoesNotExist(GetSidebandFile(result, pipB.Process), "Sideband file should not be produced for pips without any SOD outputs"); }
public void NewFileWhenParentFolderIsNotMaterialzedBeforeOperation(RepoConfig repoCfg) { using var helper = Clone(repoCfg); // track a file that doesn't exist var file = helper.GetPath(@"src\files\subfolder\newfile2.txt"); var result = helper.TrackPath(file); XAssert.AreEqual(PathExistence.Nonexistent, result.Existence); // switch to a new branch where that file does exist using var reseter = helper.GitCheckout("newFileInSubfolder"); // immediately snap changes helper.SnapCheckPoint(); // assert that 'CreateFile' USN entry was recorded helper.AssertCreateFile(file); XAssert.FileExists(file); }
public void NewFileWhenParentFolderIsMaterialzedBeforeOperation(RepoConfig repoCfg) { var helper = Clone(repoCfg); // track a file that doesn't exist var file = helper.GetPath(@"src\files\subfolder\newfile.txt"); var file2 = helper.GetPath(@"src\files\subfolder\newfile2.txt"); helper.TrackPath(file); helper.TrackPath(file2); // materialize files before git checkout helper.AssertFileOnDisk(file); helper.AssertFileOnDisk(file2, expectExists: false); using var reseter = helper.GitCheckout("newFileInSubfolder"); // snap changes helper.SnapCheckPoint(); helper.AssertCreateFile(file2); XAssert.FileExists(file2); }
public void BasicFunctionalityTest() { var logDir = Path.Combine(TemporaryDirectory, nameof(TextLoggerTests), nameof(BasicFunctionalityTest)); var logFileName = "test.log"; var port = "60000"; var infoMessage = "Info"; var debugMessage = "Debug"; var errorMessage = "Error"; var warningMessage = "Warning"; var message = "none"; Directory.CreateDirectory(logDir); // create a logger and log a couple of messages string logFileFullPath = Path.Combine(logDir, logFileName + $"-{port}.log"); using (var logger = PluginLogUtils.GetLogger <TextLoggerTests>(logDir, logFileName, port)) { logger.Info(message); logger.Warning(message); logger.Error(message); logger.Debug(message); } // check that the log file was produced XAssert.FileExists(logFileFullPath); // check that the verbose message was not logged unless 'logVerbose' is true var logLines = File.ReadAllLines(logFileFullPath); // check that every line contains the prefix; XAssert.All(logLines, line => line.Contains(nameof(TextLoggerTests))); // check individual log messages XAssert.Contains(logLines[0], infoMessage); XAssert.Contains(logLines[1], warningMessage); XAssert.Contains(logLines[2], errorMessage); XAssert.Contains(logLines[3], debugMessage); }
public void StdFileCopyTest(global::BuildXL.Utilities.Configuration.OutputReportingMode outputReportingMode) { EventListener.NestedLoggerHandler += eventData => { if (eventData.EventId == (int)LogEventId.PipProcessError) { var loggedMessage = eventData.Payload.ToArray()[5].ToString(); var extraLoggedMessage = eventData.Payload.ToArray()[6].ToString(); m_loggedPipFailures.Add(loggedMessage); m_loggedPipFailures.Add(extraLoggedMessage); } }; Configuration.Sandbox.OutputReportingMode = outputReportingMode; var text = @" * BEFORE * * <error> * * err1 * * </error> * * AFTER * * <error>err2</error> * <error>err3</error> * "; var errRegex = "error"; var ops = SplitLines(text) .Select(l => Operation.Echo(l, true)) .Concat(new[] { Operation.WriteFile(CreateOutputFileArtifact()), Operation.Fail() }); var pipBuilder = CreatePipBuilder(ops); pipBuilder.ErrorRegex = new RegexDescriptor(StringId.Create(Context.StringTable, errRegex), RegexOptions.None); pipBuilder.EnableMultiLineErrorScanning = false; Process pip = SchedulePipBuilder(pipBuilder).Process; var runResult = RunScheduler(); runResult.AssertFailure(); AssertErrorEventLogged(LogEventId.PipProcessError); string expectedPrintedError; var stdFilePathInLogDir = Path.Combine(runResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), SandboxedProcessPipExecutor.StdOutputsDirNameInLog, pip.FormattedSemiStableHash, SandboxedProcessFile.StandardError.DefaultFileName()); if (outputReportingMode == global::BuildXL.Utilities.Configuration.OutputReportingMode.TruncatedOutputOnError) { expectedPrintedError = @" * <error> * * </error> * * <error>err2</error> * <error>err3</error> * This message has been filtered by a regex. Please find the complete stdout/stderr in the following file(s) in the log directory."; XAssert.FileExists(stdFilePathInLogDir, $"StandardError file {stdFilePathInLogDir} should had been copied to log directory"); } else { expectedPrintedError = @" * <error> * * </error> * * <error>err2</error> * <error>err3</error> * This message has been filtered by a regex. Please find the complete stdout/stderr in the following file(s) or in the DX0066 event in the log file."; XAssert.FileDoesNotExist(stdFilePathInLogDir, $"StandardError file {stdFilePathInLogDir} should had been copied to log directory"); } XAssert.ArrayEqual( SplitLines(expectedPrintedError), m_loggedPipFailures.SelectMany(SplitLines).ToArray()); // Rerun the pip and the std file will be copied to the log directory too runResult = RunScheduler(); runResult.AssertFailure(); string extension = Path.GetExtension(stdFilePathInLogDir); var basename = stdFilePathInLogDir.Substring(0, stdFilePathInLogDir.Length - extension.Length); stdFilePathInLogDir = $"{basename}.1{extension}"; if (outputReportingMode == global::BuildXL.Utilities.Configuration.OutputReportingMode.TruncatedOutputOnError) { XAssert.FileExists(stdFilePathInLogDir, $"StandardError file {stdFilePathInLogDir} should had been copied to log directory"); } else { XAssert.FileDoesNotExist(stdFilePathInLogDir, $"StandardError file {stdFilePathInLogDir} should had been copied to log directory"); } AssertErrorEventLogged(LogEventId.PipProcessError); }
public void BasicFunctionalityTest(bool logVerbose) { var logDir = Path.Combine(TemporaryDirectory, nameof(FileLoggerTests), nameof(BasicFunctionalityTest)); var logFileName = "test.log"; var monikerId = "moniker"; var prefix = "QWERTY"; var infoMessage = "imessage"; var warningMessage = "wmessage"; var errorMessage = "emessage"; var verboseMessage = "vmessage"; Directory.CreateDirectory(logDir); // create a logger and log a couple of messages string logFileFullPath = null; using (var logger = new FileLogger(logDir, logFileName, monikerId, logVerbose, prefix)) { XAssert.AreEqual(logVerbose, logger.IsLoggingVerbose); XAssert.AreEqual(prefix, logger.Prefix); logger.Info(infoMessage); logger.Warning(warningMessage); logger.Error(errorMessage); logger.Verbose(verboseMessage); logFileFullPath = logger.LogFilePath; } // check that the log file was produced XAssert.FileExists(logFileFullPath); // check that the verbose message was not logged unless 'logVerbose' is true var logLines = File.ReadAllLines(logFileFullPath); XAssert.AreEqual(logVerbose ? 4 : 3, logLines.Length); // check that every line contains the prefix; XAssert.All(logLines, line => line.Contains(prefix)); // check individual log messages XAssert.Contains(logLines[0], infoMessage); XAssert.Contains(logLines[1], warningMessage); XAssert.Contains(logLines[2], errorMessage); if (logVerbose) { XAssert.Contains(logLines[3], verboseMessage); } // create the same logger and assert that it's not going to overwrite the the old log file string logFile2FullPath = null; using (var logger2 = new FileLogger(logDir, logFileName, monikerId, logVerbose, prefix)) { XAssert.AreNotEqual(logFileFullPath, logger2.LogFilePath); logger2.Log(LogLevel.Info, "hi"); logFile2FullPath = logger2.LogFilePath; } XAssert.FileExists(logFileFullPath); XAssert.FileExists(logFile2FullPath); }