private void ReparsePointWithSymlinkTest(string targetPath, string reparsePoint, bool useDirTranslation) { const string TARGET_NAME = "targetName"; const string TARGET_SYM_NAME = "targetSymFile"; // Target file artifacts FileArtifact targetFile = CreateFileArtifactWithName(TARGET_NAME, targetPath); WriteSourceFile(targetFile); FileArtifact symlinkFileOnTarget = CreateFileArtifactWithName(TARGET_SYM_NAME, targetPath); // Create a symlink file on target XAssert.PossiblySucceeded(FileUtilities.TryCreateSymbolicLink(ArtifactToString(symlinkFileOnTarget), ArtifactToString(targetFile), isTargetFile: true)); // junction file artifacts to read in pipA FileArtifact symlinkFileOnReparsePoint = CreateFileArtifactWithName(TARGET_SYM_NAME, reparsePoint); // ........... PIP A ........... var builderA = CreatePipBuilder(new Operation[] { Operation.ReadFile(symlinkFileOnReparsePoint, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact(targetPath)) }); builderA.AddInputFile(symlinkFileOnTarget); builderA.AddInputFile(targetFile); var pipA = SchedulePipBuilder(builderA); if (useDirTranslation) { string junctionPathStr = reparsePoint; string targetPathStr = targetPath; // ............CREATING DIRECTORY TRANSLATOR.................... // Before scheduling, we need to: // (1) init the directory translator // (2) add any translations // (3) then seal // - bullets 1 and 2 are happening for us now in the SchedulerIntegrationTestBase DirectoryTranslator.AddTranslation(junctionPathStr, targetPathStr); RunScheduler().AssertSuccess(); } else { // No directory translation will cause an error RunScheduler().AssertFailure(); AssertVerboseEventLogged(LogEventId.DependencyViolationMissingSourceDependency); AssertVerboseEventLogged(ProcessesLogEventId.PipProcessDisallowedFileAccess); AssertWarningEventLogged(LogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations); AssertErrorEventLogged(LogEventId.FileMonitoringError); } }
public TestScheduler( PipGraph graph, TestPipQueue pipQueue, PipExecutionContext context, FileContentTable fileContentTable, EngineCache cache, IConfiguration configuration, FileAccessWhitelist fileAccessWhitelist, DirectoryMembershipFingerprinterRuleSet directoryMembershipFingerprinterRules = null, ITempCleaner tempCleaner = null, PipRuntimeTimeTable runningTimeTable = null, JournalState journalState = null, PerformanceCollector performanceCollector = null, string fingerprintSalt = null, PreserveOutputsInfo?previousInputsSalt = null, IEnumerable <Pip> successfulPips = null, IEnumerable <Pip> failedPips = null, LoggingContext loggingContext = null, IIpcProvider ipcProvider = null, DirectoryTranslator directoryTranslator = null, VmInitializer vmInitializer = null, SchedulerTestHooks testHooks = null) : base(graph, pipQueue, context, fileContentTable, cache, configuration, fileAccessWhitelist, loggingContext, null, directoryMembershipFingerprinterRules, tempCleaner, AsyncLazy <PipRuntimeTimeTable> .FromResult(runningTimeTable), performanceCollector, fingerprintSalt, previousInputsSalt, ipcProvider: ipcProvider, directoryTranslator: directoryTranslator, journalState: journalState, vmInitializer: vmInitializer, testHooks: testHooks) { m_testPipQueue = pipQueue; if (successfulPips != null) { foreach (var pip in successfulPips) { Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler"); m_overridePipResults.Add(pip.PipId, PipResultStatus.Succeeded); } } if (failedPips != null) { foreach (var pip in failedPips) { Contract.Assume(pip.PipId.IsValid, "Override results must be added after the pip has been added to the scheduler"); m_overridePipResults.Add(pip.PipId, PipResultStatus.Failed); } } m_loggingContext = loggingContext; }
/// <summary> /// Creates an instance of <see cref="InputChangeList"/> from a stream reader. /// </summary> public static InputChangeList CreateFromStream( LoggingContext loggingContext, TextReader reader, string filePathOrigin = null, string sourceRoot = null, DirectoryTranslator directoryTranslator = null) { Contract.Requires(loggingContext != null); Contract.Requires(reader != null); InputChangeList inputChangeList = new InputChangeList(); try { int lineNo = 0; string inputLine = null; while ((inputLine = reader.ReadLine()) != null) { ++lineNo; if (string.IsNullOrEmpty(inputLine)) { continue; } if (!TryParseInput( loggingContext, inputLine, filePathOrigin ?? string.Empty, lineNo, out var changePathInfo, sourceRoot, directoryTranslator)) { return(null); } inputChangeList.m_changedPaths.Add(changePathInfo); } } catch (IOException ioException) { Logger.Log.ExceptionOnCreatingInputChangeList(loggingContext, filePathOrigin ?? string.Empty, ioException.ToString()); return(null); } return(inputChangeList); }
public void TestDirectoryTranslatorNoCycle2() { var context = BuildXLContext.CreateInstanceForTesting(); var pathTable = context.PathTable; DirectoryTranslator.RawInputTranslation[] translations = new[] { CreateInputTranslation(pathTable, getAtoms(new string[] { "d", "foo", "bar" }), getAtoms(new string[] { "E" })), CreateInputTranslation(pathTable, getAtoms(new string[] { "A" }), getAtoms(new string[] { "B" })), CreateInputTranslation(pathTable, getAtoms(new string[] { "C" }), getAtoms(new string[] { "D" })), CreateInputTranslation(pathTable, getAtoms(new string[] { "B" }), getAtoms(new string[] { "C" })) }; string error; XAssert.IsTrue(DirectoryTranslator.ValidateDirectoryTranslation(pathTable, translations, out error)); }
public void UseJunctionRoots() { AbsolutePath targetPath = CreateUniqueDirectory(SourceRoot); AbsolutePath junctionPath = CreateUniqueDirectory(SourceRoot); string targetPathStr = targetPath.ToString(Context.PathTable); string junctionPathStr = junctionPath.ToString(Context.PathTable); // .......... Creating the Junction .......... FileUtilities.CreateJunction(junctionPath.ToString(Context.PathTable), targetPathStr); FileArtifact sourceFile = CreateSourceFile(junctionPathStr); Configuration.Engine.DirectoriesToTranslate.Add( new BuildXLConfiguration.TranslateDirectoryData( targetPath.ToString(Context.PathTable) + @"\<" + junctionPath.ToString(Context.PathTable) + @"\", targetPath, junctionPath)); DirectoryTranslator.AddTranslation(targetPath, junctionPath, Context.PathTable); var pipBuilder = CreatePipBuilder(new Operation[] { Operation.ReadFile(sourceFile), Operation.WriteFile(CreateOutputFileArtifact()) }); SchedulePipBuilder(pipBuilder); RunScheduler().AssertSuccess(); // Remove junction and recreate one with the same target AssertTrue(FileUtilities.TryRemoveDirectory(junctionPathStr, out var hr)); Directory.CreateDirectory(junctionPathStr); FileUtilities.CreateJunction(junctionPath.ToString(Context.PathTable), targetPathStr); RunScheduler().AssertSuccess(); AssertVerboseEventLogged(EventId.ValidateJunctionRoot); AssertVerboseEventLogged(StorageLogEventId.IgnoredRecordsDueToUnchangedJunctionRootCount); // Remove junction and recreate one with the same target AssertTrue(FileUtilities.TryRemoveDirectory(junctionPathStr, out var hr2)); Directory.CreateDirectory(junctionPathStr); FileUtilities.CreateJunction(junctionPath.ToString(Context.PathTable), targetPathStr); RunScheduler().AssertSuccess(); AssertVerboseEventLogged(EventId.ValidateJunctionRoot); AssertVerboseEventLogged(StorageLogEventId.IgnoredRecordsDueToUnchangedJunctionRootCount); }
public void TestInvalidDirectoryTranslatorDueToCycle() { var context = BuildXLContext.CreateInstanceForTesting(); var pathTable = context.PathTable; DirectoryTranslator.RawInputTranslation[] translations = new[] { CreateInputTranslation(pathTable, getAtoms(new string[] { "d", "foo", "bar" }), getAtoms(new string[] { "E" })), CreateInputTranslation(pathTable, getAtoms(new string[] { "A" }), getAtoms(new string[] { "B" })), CreateInputTranslation(pathTable, getAtoms(new string[] { "C" }), getAtoms(new string[] { "A" })), CreateInputTranslation(pathTable, getAtoms(new string[] { "B" }), getAtoms(new string[] { "C" })) }; string error; XAssert.IsFalse(DirectoryTranslator.ValidateDirectoryTranslation(pathTable, translations, out error)); XAssert.AreEqual(@"cycle in directory translations '" + A(getAtoms(new string[] { "A" })) + "' < '" + A(getAtoms(new string[] { "B" })) + "' < '" + A(getAtoms(new string[] { "C" })) + "' < '" + A(getAtoms(new string[] { "A" })) + "'", error); }
public void TestDirectoryTranslatorWithJunctionCreation(bool createSource, bool createTarget, bool createJunction) { var context = BuildXLContext.CreateInstanceForTesting(); var pathTable = context.PathTable; DirectoryTranslator.RawInputTranslation translation; string dir = System.Guid.NewGuid().ToString(); var translations = new[] { translation = CreateInputTranslationWithJunction(pathTable, new[] { "Source", "S__" + dir }, new[] { "Target", "T__" + dir }, createSource, createTarget, createJunction) }; bool result = DirectoryTranslator.TestForJunctions(pathTable, translations, out string error); if (createJunction) { createSource = true; createTarget = true; } XAssert.AreEqual(result, createSource && createTarget && createJunction, result ? "Success" : error); if (!result) { string source = translation.SourcePath.ToString(pathTable); string target = translation.TargetPath.ToString(pathTable); XAssert.IsTrue(error.Contains($"Translation from '{source}' to '{target}':"), error); if (!createSource) { XAssert.IsTrue(error.Contains($"'{source}' does not exist"), error); } else if (!createTarget) { XAssert.IsTrue(error.Contains($"'{target}' does not exist"), error); } else if (!createJunction) { XAssert.IsTrue(error.Contains("Expect target file"), error); } } }
public void TranslateGlobalUntrackedScope(bool translate, Process.Options requireGlobalDependencies) { DirectoryArtifact sourceDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(SourceRoot, prefix: "sourceDir")); DirectoryArtifact targetDirectory = DirectoryArtifact.CreateWithZeroPartialSealId(CreateUniqueDirectory(SourceRoot, prefix: "targetDir")); FileArtifact outputFileInTargetDir = CreateOutputFileArtifact(ArtifactToString(targetDirectory)); FileArtifact inputFileInTargetDir = CreateSourceFile(ArtifactToString(targetDirectory)); Configuration.Sandbox.GlobalUnsafeUntrackedScopes.Add(sourceDirectory); if (translate) { DirectoryTranslator = new DirectoryTranslator(); DirectoryTranslator.AddTranslation(ArtifactToString(sourceDirectory), ArtifactToString(targetDirectory)); } var ops = new Operation[] { Operation.ReadFile(inputFileInTargetDir, doNotInfer: true), Operation.WriteFile(outputFileInTargetDir) }; var builder = CreatePipBuilder(ops); builder.Options |= requireGlobalDependencies; Process pip = SchedulePipBuilder(builder).Process; if (translate && ((requireGlobalDependencies & Process.Options.RequireGlobalDependencies) == Process.Options.RequireGlobalDependencies)) { RunScheduler().AssertCacheMiss(pip.PipId); RunScheduler().AssertCacheHit(pip.PipId); } else { RunScheduler().AssertFailure(); AssertWarningEventLogged(EventId.ProcessNotStoredToCacheDueToFileMonitoringViolations); AssertErrorEventLogged(EventId.FileMonitoringError); } }
public void ChangeJunctionWithTranslateDirectoryShouldRebuild() { const string ReadFileName = "file"; AbsolutePath targetDirectoryA = CreateUniqueDirectory(SourceRoot, "DirA"); AbsolutePath targetDirectoryB = CreateUniqueDirectory(SourceRoot, "DirB"); AbsolutePath targetDirectoryC = CreateUniqueDirectory(SourceRoot, "DirC"); string expandedTargetDirectoryA = targetDirectoryA.ToString(Context.PathTable); string expandedTargetDirectoryB = targetDirectoryB.ToString(Context.PathTable); string expandedTargetDirectoryC = targetDirectoryC.ToString(Context.PathTable); AbsolutePath junction1 = CreateUniqueDirectory(SourceRoot, "Junc1"); AbsolutePath junction2 = CreateUniqueDirectory(SourceRoot, "Junc2"); string expandedJunction1 = junction1.ToString(Context.PathTable); string expandedJunction2 = junction2.ToString(Context.PathTable); FileArtifact targetAFile = CreateFileArtifactWithName(ReadFileName, expandedTargetDirectoryA); FileArtifact targetBFile = CreateFileArtifactWithName(ReadFileName, expandedTargetDirectoryB); FileArtifact targetCFile = CreateFileArtifactWithName(ReadFileName, expandedTargetDirectoryC); WriteSourceFile(targetAFile); WriteSourceFile(targetBFile); WriteSourceFile(targetCFile); FileArtifact junction1File = CreateFileArtifactWithName(ReadFileName, expandedJunction1); FileArtifact junction2File = CreateFileArtifactWithName(ReadFileName, expandedJunction2); // Create junction // Junc1 -> DirA // Junc2 -> DirC FileUtilities.CreateJunction(expandedJunction1, expandedTargetDirectoryA); FileUtilities.CreateJunction(expandedJunction2, expandedTargetDirectoryC); FileArtifact outputP = CreateOutputFileArtifact(); FileArtifact outputQ = CreateOutputFileArtifact(); var builderP = CreatePipBuilder(new Operation[] { Operation.CopyFile(junction1File, outputP) }); var pipP = SchedulePipBuilder(builderP).Process; var builderQ = CreatePipBuilder(new Operation[] { Operation.CopyFile(targetCFile, outputQ, doNotInfer: true) }); builderQ.AddInputFile(junction2File); builderQ.AddOutputFile(outputQ.Path, FileExistence.Required); var pipQ = SchedulePipBuilder(builderQ).Process; DirectoryTranslator = new DirectoryTranslator(); DirectoryTranslator.AddTranslation(expandedTargetDirectoryA, expandedJunction1); DirectoryTranslator.AddTranslation(expandedTargetDirectoryC, expandedJunction2); RunScheduler().AssertSuccess().AssertCacheMiss(pipP.PipId, pipQ.PipId); // Change junction // Junc2 -> DirB FileUtilities.CreateJunction(expandedJunction1, expandedTargetDirectoryB); var result = RunScheduler().AssertSuccess().AssertCacheMiss(pipP.PipId); if (Configuration.Schedule.IncrementalScheduling) { // pipQ should not be affected at all. result.AssertNotScheduled(pipQ.PipId); } else { // pipQ is scheduled but results in cache hit. result.AssertScheduled(pipQ.PipId).AssertCacheHit(pipQ.PipId); } }
public void ChangeJunctionShouldRebuild() { const string ReadFileName = "file"; AbsolutePath targetDirectoryA = CreateUniqueDirectory(SourceRoot, "DirA"); AbsolutePath targetDirectoryB = CreateUniqueDirectory(SourceRoot, "DirB"); string expandedTargetDirectoryA = targetDirectoryA.ToString(Context.PathTable); string expandedTargetDirectoryB = targetDirectoryB.ToString(Context.PathTable); AbsolutePath junction1 = CreateUniqueDirectory(SourceRoot, "Junc1"); string expandedJunction1 = junction1.ToString(Context.PathTable); FileArtifact targetAFile = CreateFileArtifactWithName(ReadFileName, expandedTargetDirectoryA); FileArtifact targetBFile = CreateFileArtifactWithName(ReadFileName, expandedTargetDirectoryB); WriteSourceFile(targetAFile); WriteSourceFile(targetBFile); FileArtifact junction1File = CreateFileArtifactWithName(ReadFileName, expandedJunction1); // Create junction // Junc1 -> DirA FileUtilities.CreateJunction(expandedJunction1, expandedTargetDirectoryA); FileArtifact outputP = CreateOutputFileArtifact(); FileArtifact outputQ = CreateOutputFileArtifact(); var pipP = SchedulePipBuilder(CreatePipBuilder(new Operation[] { Operation.CopyFile(junction1File, outputP) })).Process; // We don't need directory translator here because pipP reads through junction1File. // But we want to show that incremental scheduling is not sensitive to directory translator. DirectoryTranslator = new DirectoryTranslator(); DirectoryTranslator.AddTranslation(expandedTargetDirectoryA, expandedJunction1); RunScheduler().AssertSuccess().AssertCacheMiss(pipP.PipId); // Change junction // Junc1 -> DirB FileUtilities.CreateJunction(expandedJunction1, expandedTargetDirectoryB); DirectoryTranslator = new DirectoryTranslator(); DirectoryTranslator.AddTranslation(expandedTargetDirectoryB, expandedJunction1); RunScheduler().AssertSuccess().AssertCacheMiss(pipP.PipId); // Modify DirB\File WriteSourceFile(targetBFile); RunScheduler().AssertSuccess().AssertCacheMiss(pipP.PipId); // Modify DirA\File WriteSourceFile(targetAFile); // Note that, even though DirA\File has nothing to do with the build as the junction has changed to DirB, // incremental scheduling, when enabled, will still mark pipP dirty. In the first build the file change tracker // tracked DirA\File, and introduced a mapping from FileId(DirA\File) to Path(Junc1\File). Thus, when DirA\File // changes, Path(Junc1\File) is affected, and since pipP specifies Path(Junc1\File) as its input, pipP is affected as well. RunScheduler().AssertSuccess().AssertScheduled(pipP.PipId).AssertCacheHit(pipP.PipId); // Modify again DirA\File. // After the above run, the mapping FileId(DirA\File) to Path(Junc1\File) has been removed, and thus, // any change to DirA\File should not affect pipP. WriteSourceFile(targetAFile); var result = RunScheduler().AssertSuccess(); if (Configuration.Schedule.IncrementalScheduling) { result.AssertNotScheduled(pipP.PipId); } else { result.AssertCacheHit(pipP.PipId); } }
/// <summary> /// Constructor. /// </summary> public RootTranslator() { m_translator = new DirectoryTranslator(); }
/// <summary> /// Constructor. /// </summary> public RootTranslator(DirectoryTranslator translator) { m_translator = translator; }
/// <summary> /// Tries to parse input line. /// </summary> /// <remarks> /// Input line must be of the form: /// full path /// or /// full path|comma separated <see cref="PathChanges"/> /// The former assumes that changes are <see cref="PathChanges.DataOrMetadataChanged"/>. /// </remarks> private static bool TryParseInput( LoggingContext loggingContext, string input, string filePath, int lineNo, out ChangedPathInfo changedPathInfo, string sourceRoot = null, DirectoryTranslator directoryTranslator = null) { Contract.Requires(loggingContext != null); Contract.Requires(!string.IsNullOrEmpty(input)); changedPathInfo = default; string[] splitInput = input.Split(s_inputSeparator); if (splitInput.Length > 2) { Logger.Log.InvalidFormatOfInputChange(loggingContext, input, filePath, lineNo); return(false); } string changedPath = splitInput[0].Trim(); string changesStr = null; if (splitInput.Length == 2) { changesStr = splitInput[1].Trim(); } // Assume data or metadata change if unspecified. PathChanges changes = PathChanges.DataOrMetadataChanged; try { if (!Path.IsPathRooted(changedPath)) { if (string.IsNullOrEmpty(sourceRoot)) { Logger.Log.InvalidChangedPathOfInputChange(loggingContext, changedPath, filePath, lineNo); return(false); } changedPath = Path.GetFullPath(Path.Combine(sourceRoot, changedPath)); } if (directoryTranslator != null) { changedPath = directoryTranslator.Translate(changedPath); } if (!string.IsNullOrEmpty(changesStr)) { if (!Enum.TryParse(changesStr, true, out changes)) { string validKinds = string.Join(", ", ((PathChanges[])Enum.GetValues(typeof(PathChanges))).Select(c => c.ToString())); Logger.Log.InvalidChangeKindsOfInputChange(loggingContext, changesStr, filePath, lineNo, validKinds); return(false); } } } catch (ArgumentException argumentException) { Logger.Log.InvalidInputChange(loggingContext, input, filePath, lineNo, argumentException.ToString()); return(false); } changedPathInfo = new ChangedPathInfo(changedPath, changes); return(true); }
/// <nodoc/> public SymlinkedAccessResolver(PipExecutionContext context, [CanBeNull] DirectoryTranslator directoryTranslator) { Contract.RequiresNotNull(context); m_context = context; m_directoryTranslator = directoryTranslator; }
/// <summary> /// Creates an instance of <see cref="FrontEndEngineImplementation"/>. /// </summary> public FrontEndEngineImplementation( LoggingContext loggingContext, PathTable pathTable, IConfiguration configuration, IStartupConfiguration startupConfiguration, MountsTable mountsTable, InputTracker inputTracker, SnapshotCollector snapshotCollector, DirectoryTranslator directoryTranslator, Func <FileContentTable> getFileContentTable, int timerUpdatePeriod, bool isPartialReuse, IEnumerable <IFrontEnd> registeredFrontends) { Contract.Requires(loggingContext != null); Contract.Requires(pathTable != null); Contract.Requires(configuration != null); Contract.Requires(startupConfiguration != null); Contract.Requires(mountsTable != null); Contract.Requires(inputTracker != null); Contract.Requires(getFileContentTable != null); Contract.Requires(registeredFrontends != null); m_loggingContext = loggingContext; PathTable = pathTable; m_mountsTable = mountsTable; m_inputTracker = inputTracker; m_getFileContentTable = getFileContentTable; m_isPartialReuse = isPartialReuse; m_frontendsEnvironmentRestriction = registeredFrontends.ToDictionary(frontend => frontend.Name, frontEnd => frontEnd.ShouldRestrictBuildParameters); m_snapshotCollector = snapshotCollector; GetTimerUpdatePeriod = timerUpdatePeriod; Layout = configuration.Layout; if (ShouldUseSpecCache(configuration)) { m_specCache = new FileCombiner( loggingContext, Path.Combine(configuration.Layout.EngineCacheDirectory.ToString(PathTable), SpecCacheFileName), FileCombiner.FileCombinerUsage.SpecFileCache, configuration.FrontEnd.LogStatistics); } m_allBuildParameters = new ConcurrentDictionary <string, TrackedValue>(StringComparer.OrdinalIgnoreCase); foreach (var kvp in PopulateFromEnvironmentAndApplyOverrides(loggingContext, startupConfiguration.Properties).ToDictionary()) { m_allBuildParameters.TryAdd(kvp.Key, new TrackedValue(kvp.Value, false)); } m_localDiskContentStore = new LocalDiskContentStore( loggingContext, PathTable, m_getFileContentTable(), m_inputTracker.FileChangeTracker, directoryTranslator, vfsCasRoot: configuration.Cache.VfsCasRoot); m_localDiskContentStoreConcurrencyLimiter = new ActionBlockSlim <MaterializeFileRequest>( Environment.ProcessorCount, request => { var requestCompletionSource = request.CompletionSource; try { var materializeResult = m_localDiskContentStore.TryMaterializeAsync( request.Cache, request.FileRealizationModes, request.Path, request.ContentHash, trackPath: request.TrackPath, recordPathInFileContentTable: request.RecordPathInFileContentTable).GetAwaiter().GetResult(); requestCompletionSource.SetResult(materializeResult); } catch (TaskCanceledException) { requestCompletionSource.SetCanceled(); } catch (Exception e) { requestCompletionSource.SetException(e); } }); }
/// <nodoc/> public ReparsePointResolver(PipExecutionContext context, [CanBeNull] DirectoryTranslator directoryTranslator) { Contract.RequiresNotNull(context); m_context = context; m_directoryTranslator = directoryTranslator; }