public void TestDumpPipLiteWithPassingAndFailingPips() { var passingCopyFile = new CopyFile(CreateSourceFile(), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context)); var failingCopyFile = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context)); PipGraphBuilder.AddCopyFile(passingCopyFile); PipGraphBuilder.AddCopyFile(failingCopyFile); var schedulerResult = RunScheduler().AssertFailure(); var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips"); var failingPipDumpFile = Path.Combine(logFolder, $"{failingCopyFile.FormattedSemiStableHash}.json"); var passingPipDumpFile = Path.Combine(logFolder, $"{passingCopyFile.FormattedSemiStableHash}.json"); List <Option> options = new List <Option> { new Option() { Name = "d+" } }; RunAnalyzer(schedulerResult, null, options); Assert.True(Directory.Exists(logFolder)); Assert.True(File.Exists(failingPipDumpFile)); Assert.True(!File.Exists(passingPipDumpFile)); }
public void AffectedSourceInsideSourceSealDirectory() { var sourceDirectory = CreateUniqueDirectory(SourceRootPath); var fileInsideSourceDirectory = CreateSourceFile(sourceDirectory, "file_"); var sourceSealDirectory = CreateSourceSealDirectory(sourceDirectory, SealDirectoryKind.SourceTopDirectoryOnly, "file_*"); PipGraphBuilder.AddSealDirectory(sourceSealDirectory); var changeAffectedWrittenFile = CreateUniqueObjPath("change"); var pipBuilder = CreatePipBuilder(new Operation[] { // Ensure that pip reads the changeAffectedWrittenFile if it exists. Operation.ReadFile(FileArtifact.CreateSourceFile(changeAffectedWrittenFile), doNotInfer: true), Operation.ReadFile(fileInsideSourceDirectory, doNotInfer: true), Operation.WriteFile(CreateOutputFileArtifact()), }); pipBuilder.AddInputDirectory(sourceSealDirectory.Directory); pipBuilder.SetChangeAffectedInputListWrittenFile(changeAffectedWrittenFile); SchedulePipBuilder(pipBuilder); var inputChangesFile = CreateOutputFileArtifact(); File.WriteAllText(ArtifactToString(inputChangesFile), ArtifactToString(fileInsideSourceDirectory)); Configuration.Schedule.InputChanges = inputChangesFile; RunScheduler().AssertSuccess(); var actualAffectedInput = File.ReadAllText(changeAffectedWrittenFile.ToString(Context.PathTable)); XAssert.AreEqual(ArtifactToString(fileInsideSourceDirectory), actualAffectedInput); }
public void TestDumpPipLite() { var copyFile = new CopyFile(CreateSourceFile(), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context)); PipGraphBuilder.AddCopyFile(copyFile); var schedulerResult = RunScheduler().AssertSuccess(); var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips"); var pipDumpFile = Path.Combine(logFolder, $"{copyFile.FormattedSemiStableHash}.json"); List <Option> options = new List <Option> { new Option() { Name = "o", Value = schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable) }, new Option() { Name = "p", Value = copyFile.FormattedSemiStableHash } }; RunAnalyzer(schedulerResult, null, options); Assert.True(Directory.Exists(logFolder)); Assert.True(File.Exists(pipDumpFile)); }
public void TestFailingPipDump(bool failPip) { BaseSetup(); FileArtifact sourceArtifact = failPip ? FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)) : CreateSourceFile(); FileArtifact destinationArtifact = CreateOutputFileArtifact(); var copyFile = CreateCopyFile(sourceArtifact, destinationArtifact); PipGraphBuilder.AddCopyFile(copyFile); var schedulerResult = RunScheduler(); var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips"); var pipDumpFile = Path.Combine(logFolder, $"{copyFile.FormattedSemiStableHash}.json"); if (failPip) { schedulerResult.AssertFailure(); SetExpectedFailures(1, 0); // One error logged for the failing pip AssertErrorCount(); Assert.True(Directory.Exists(logFolder)); Assert.True(File.Exists(pipDumpFile)); Assert.True(new FileInfo(pipDumpFile).Length > 0); //Ensure that some content was written to the file } else { schedulerResult.AssertSuccess(); // FailedPips log folder should not have been created when we run successfully Assert.False(Directory.Exists(logFolder)); Assert.False(File.Exists(pipDumpFile)); } }
public void FailWhenTwoPipsHaveSameTempDirectory(bool allowDuplicateTemporaryDirectory, int tempArtifactType) { Configuration.Engine.AllowDuplicateTemporaryDirectory = allowDuplicateTemporaryDirectory; Exception caughtExpection = null; try { CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut1); CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut2); PipGraphBuilder.Build(); } catch (Exception e) { caughtExpection = e; } if (!allowDuplicateTemporaryDirectory) { AssertTrue(caughtExpection != null, "Expected PipGraph Builder Exception, but Graph built successfully"); AssertErrorEventLogged(LogEventId.MultiplePipsUsingSameTemporaryDirectory); } else { AssertTrue(caughtExpection == null, "Expected PipGraph Builder to pass, but Graph built failed with exception: " + caughtExpection?.Message); } }
private void RunAndAssertDumpPip(Pip pip) { var graph = PipGraphBuilder.Build(); var success = CreateLogPathAndRun(pip, graph); AssertCommon(success, pip, graph); }
public void ConsumeFilterPassingFile(bool topOnly) { FileArtifact source; if (topOnly) { source = FileArtifact.CreateSourceFile(SourceRootPath.Combine(Context.PathTable, "file.txt")); } else { var nestedDir = AbsolutePath.Create(Context.PathTable, Path.Combine(SourceRoot, "nested")); source = FileArtifact.CreateSourceFile(nestedDir.Combine(Context.PathTable, "file.txt")); } WriteSourceFile(source); var output = CreateOutputFileArtifact(ObjectRoot); SealDirectory sealedDirectory = CreateSourceSealDirectory(SourceRootPath, topOnly ? SealDirectoryKind.SourceTopDirectoryOnly : SealDirectoryKind.SourceAllDirectories, "*.txt", "*.cs"); DirectoryArtifact dir = PipGraphBuilder.AddSealDirectory(sealedDirectory); var builder = CreatePipBuilder(new Operation[] { Operation.ReadFile(source, doNotInfer: true), Operation.WriteFile(output), }); builder.AddInputDirectory(dir); SchedulePipBuilder(builder); RunScheduler().AssertSuccess(); }
public void ConsumeFilterNotPassingFile() { FileArtifact source = FileArtifact.CreateSourceFile(SourceRootPath.Combine(Context.PathTable, "file.txt")); WriteSourceFile(source); var output = CreateOutputFileArtifact(ObjectRoot); SealDirectory sealedDirectory = CreateSourceSealDirectory(SourceRootPath, SealDirectoryKind.SourceTopDirectoryOnly, "*.cs"); DirectoryArtifact dir = PipGraphBuilder.AddSealDirectory(sealedDirectory); var builder = CreatePipBuilder(new Operation[] { Operation.ReadFile(source, doNotInfer: true), Operation.WriteFile(output), }); builder.AddInputDirectory(dir); SchedulePipBuilder(builder); RunScheduler().AssertFailure(); AssertVerboseEventLogged(ProcessesLogEventId.PipProcessDisallowedFileAccess); AssertVerboseEventLogged(LogEventId.DependencyViolationMissingSourceDependency); AssertWarningEventLogged(LogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations); AssertErrorEventLogged(LogEventId.FileMonitoringError); }
public void TestWriteFileDump() { FileArtifact outputFile = CreateOutputFileArtifact(); WriteFile pip = CreateWriteFile(outputFile, string.Empty, new[] { "some content" }); PipGraphBuilder.AddWriteFile(pip); RunAndAssertDumpPip(pip); }
public void TestModulePipDump() { AbsolutePath specPath = CreateUniqueSourcePath(); ModulePip module = ModulePip.CreateForTesting(Context.StringTable, specPath); PipGraphBuilder.AddModule(module); RunAndAssertDumpPip(module); }
/// <summary> /// Runs the scheduler using the instance member PipGraph and Configuration objects. This will also carry over /// any state from any previous run such as the cache /// </summary> public ScheduleRunResult RunScheduler(SchedulerTestHooks testHooks = null, SchedulerState schedulerState = null, RootFilter filter = null, TempCleaner tempCleaner = null) { if (m_graphWasModified || m_lastGraph == null) { m_lastGraph = PipGraphBuilder.Build(); XAssert.IsNotNull(m_lastGraph, "Failed to build pip graph"); } m_graphWasModified = false; return(RunSchedulerSpecific(m_lastGraph, testHooks, schedulerState, filter, tempCleaner)); }
public void FailWhenCopyFileOutputsToTemp(int tempArtifactType) { CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut); var srcFile = CreateSourceFile(); var destFile = CreateOutputFileArtifact(TempRoot); var copyFilePip = CreateAndScheduleCopyFile(srcFile, destFile); PipGraphBuilder.Build(); AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath); }
public void TestSealDirectoryDump() { var sealPath = CreateOutputDirectoryArtifact(TemporaryDirectory); SealDirectory sealDirectory = CreateSealDirectory(sealPath.Path, SealDirectoryKind.Full, scrub: true, new[] { CreateSourceFile(), CreateSourceFile(), CreateSourceFile() }); sealDirectory.SetDirectoryArtifact(sealPath); PipGraphBuilder.AddSealDirectory(sealDirectory); RunAndAssertDumpPip(sealDirectory); }
public void TestCopyFileDump() { var sourceArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, GetFullPath("source"))); var destinationArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, GetFullPath("destination"))); CopyFile pip = CreateCopyFile(sourceArtifact, destinationArtifact); PipGraphBuilder.AddCopyFile(pip); RunAndAssertDumpPip(pip); }
public void TestProcessDump() { var sourceFile = CreateSourceFile(); var outputFile = CreateOutputFileArtifact(); Process process = CreateCmdProcess(dependencies: new[] { sourceFile }, outputs: new[] { outputFile }); PipGraphBuilder.AddProcess(process); RunAndAssertDumpPip(process); }
public void FailWhenTempDirectoryContainsTempDirectory(int tempArtifactType) { var parentTemp = CreateUniqueDirectory(TempRoot); CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut, parentTemp); var childTemp = CreateUniqueDirectory(parentTemp); CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut2, childTemp); PipGraphBuilder.Build(); }
public void FailWhenTempDirectoryContainsAnotherPipsOutputFile(int tempArtifactType) { CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut); // Pip attempts to declare an output in a temp directory (that was declared by a different pip) var pipInvalidOutInTempOps = CreateAndSchedulePipBuilder(new Operation[] { Operation.WriteFile(CreateOutputFileArtifact(TempRoot)) }).Process; PipGraphBuilder.Build(); AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath); }
public void TestBadPath() { FileArtifact outputFile = CreateOutputFileArtifact(); WriteFile pip = CreateWriteFile(outputFile, string.Empty, new[] { "some content" }); PipGraphBuilder.AddWriteFile(pip); var graph = PipGraphBuilder.Build(); var success = DumpPipLiteAnalysisUtilities.DumpPip(pip, @"X:\not\a\real\path\", Context.PathTable, Context.StringTable, Context.SymbolTable, graph, LoggingContext); Assert.False(success); AssertWarningEventLogged(LogEventId.DumpPipLiteUnableToSerializePipDueToBadPath); }
/// <summary> /// Serializes and deserializes graph fragments synchronously according to its dependency relation specified by their order in the array. /// </summary> /// <param name="fragments">Graph fragments with total order on their dependency relation.</param> /// <returns>Resulting pip graph.</returns> private PipGraph SerializeDeserializeFragmentsSynchronously(params TestPipGraphFragment[] fragments) { var streams = SerializeFragmentsSynchronously(fragments); var fragmentManager = new PipGraphFragmentManager(LoggingContext, Context, PipGraphBuilder); for (int i = 0; i < streams.Length; ++i) { XAssert.IsTrue(fragmentManager.AddFragmentFileToGraph(streams[i], fragments[i].ModuleName)); } DisposeStreams(streams); return(PipGraphBuilder.Build()); }
public void TestAddSealDirectory() { var root = CreateUniqueSourcePath("root"); var seal = CreateSealDirectory(root, SealDirectoryKind.Partial); PipGraphBuilder.AddSealDirectory(seal); var builder = new PatchablePipGraph( oldPipGraph: PipGraphBuilder.DirectedGraph, oldPipTable: PipTable, graphBuilder: CreatePipGraphBuilder(), maxDegreeOfParallelism: Environment.ProcessorCount); var stats = builder.PartiallyReloadGraph(new HashSet <AbsolutePath>()); Assert.Equal(1, stats.NumPipsReloaded); }
private Pip[] CreateGraph(int numNodes, string graphAsString) { // parse edges var graph = SimpleGraph.Parse(numNodes, graphAsString); // for each node create a single output file var outFiles = Enumerable .Range(0, numNodes) .Select(_ => CreateOutputFileArtifact()) .ToArray(); // for each node create a Process pip with a single output file and dependencies according to 'edges' var processes = Enumerable .Range(0, numNodes) .Select(procIdx => { var dependencies = graph .Edges .Where(e => e.Dest == procIdx) .Select(e => outFiles[e.Src]) .ToArray(); var processBuilder = new ProcessBuilder(); var arguments = SchedulerTestBase.CreateCmdArguments( stringTable: Context.StringTable, dependencies: dependencies, outputs: new[] { outFiles[procIdx] }); return(processBuilder .WithContext(Context) .WithWorkingDirectory(GetWorkingDirectory()) .WithStandardDirectory(GetStandardDirectory()) .WithExecutable(CmdExecutable) .WithArguments(arguments) .WithDependencies(dependencies) .WithOutputs(outFiles[procIdx]) .WithProvenance(CreateProvenance()) .Build()); }) .ToArray(); // add created processes to PipGraphBuilder foreach (var proc in processes) { PipGraphBuilder.AddProcess(proc); } // return created processes return(processes); }
public void FailWhenTempDirectoryContainsAnotherPipsOutputDirectory(int tempArtifactType) { CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut); // Pip attempts to declare an output directory in a temp directory (that was declared by a different pip) var builder = CreatePipBuilder(new Operation[] { }); var opaqueDir = Path.Combine(TempRoot, "opaquedir"); var opaqueDirPath = AbsolutePath.Create(Context.PathTable, opaqueDir); builder.AddOutputDirectory(opaqueDirPath, SealDirectoryKind.Opaque); SchedulePipBuilder(builder); PipGraphBuilder.Build(); AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath); }
public ProcessWithOutputs SchedulePipBuilder(ProcessBuilder builder) { AddUntrackedWindowsDirectories(builder); if (!builder.TryFinish(PipConstructionHelper, out var process, out var outputs)) { throw new BuildXLTestException("Failed to construct process pip"); } if (!PipGraphBuilder.AddProcess(process)) { throw new BuildXLTestException("Failed to add process pip"); } return(new ProcessWithOutputs(process, outputs)); }
public void FailWhenTempDirectoryContainsSourceFile(int tempArtifactType) { CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut); // Declare a dependency on a source file within a different pip's temp directory var src = CreateSourceFile(TempRootPath); CreateAndSchedulePipBuilder(new Operation[] { Operation.ReadFile(src), Operation.WriteFile(CreateOutputFileArtifact()) }); PipGraphBuilder.Build(); AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath); }
public void FailWhenSealedSourceDirectoryIsTempArtifact(int tempArtifactType) { var sealDir = SealDirectory(SourceRootPath, SealDirectoryKind.SourceAllDirectories); var builder = CreatePipBuilder(new Operation[] { Operation.WriteFile(CreateOutputFileArtifact()) }); builder.AddInputDirectory(sealDir); SchedulePipBuilder(builder); CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut, tempRoot: sealDir.Path); PipGraphBuilder.Build(); AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath); }
public void FailWhenTempDirectoryContainsPipsOutputFile(int tempArtifactType) { var tempOut = CreateOutputFileArtifact(TempRoot); // Pip attempts to declare an output in its own temp CreateAndScheduleTempDirProcess(new Operation[] { Operation.WriteFile(tempOut), Operation.WriteFile(CreateOutputFileArtifact()) }, tempArtifactType, TempRootPath, tempOut.Path); PipGraphBuilder.Build(); AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath); }
public void FailWhenTwoPipsHaveSameTempDirectory(int tempArtifactType) { Exception exception = null; try { CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut1); CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut2); PipGraphBuilder.Build(); } catch (Exception ex) { exception = ex; } XAssert.AreNotEqual(null, exception); }
/// <summary> /// Serializes and deserializes graph fragments in parallel. /// </summary> /// <param name="fragments">Graph fragments.</param> /// <returns>Resulting pip graph.</returns> /// <remarks> /// The graph fragments, <paramref name="fragments"/>, are serialized synchronously, but are deserialized in parallel. /// For correctness, the graph fragments in <paramref name="fragments"/> are assumed to be independent of each other. /// </remarks> private PipGraph SerializeDeserializeFragmentsInParallel(params TestPipGraphFragment[] fragments) { var streams = SerializeFragmentsSynchronously(fragments); var fragmentManager = new PipGraphFragmentManager(LoggingContext, Context, PipGraphBuilder); Parallel.For( 0, fragments.Length, i => { XAssert.IsTrue(fragmentManager.AddFragmentFileToGraph(streams[i], fragments[i].ModuleName)); }); DisposeStreams(streams); return(PipGraphBuilder.Build()); }
public void TestLogLimit() { var failingCopyFile1 = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(Context)); var failingCopyFile2 = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(Context)); PipGraphBuilder.AddCopyFile(failingCopyFile1); PipGraphBuilder.AddCopyFile(failingCopyFile2); Configuration.Logging.DumpFailedPipsLogLimit = 1; var schedulerResult = RunScheduler().AssertFailure(); var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips"); SetExpectedFailures(2, 0); AssertErrorCount(); AssertVerboseEventLogged(LogEventId.RuntimeDumpPipLiteLogLimitReached, count: 1); Assert.True(Directory.GetFiles(logFolder).Length == 1); }
public void TestProcessDump() { var sourceFile = CreateSourceFile(); var outputFile = CreateOutputFileArtifact(); Process process = CreateCmdProcess(dependencies: new[] { sourceFile }, outputs: new[] { outputFile }); PipGraphBuilder.AddProcess(process); RunAndAssertDumpPip(process); // Test a few empty list fields to ensure they did not get printed out var pipWrittenToFile = Encoding.UTF8.GetString(File.ReadAllBytes(GetDumpFilePath(process))); // This pip does not have any of the following fields set (they will have size 0 lists) Assert.False(pipWrittenToFile.Contains("Tags")); Assert.False(pipWrittenToFile.Contains("Environment Variables")); Assert.False(pipWrittenToFile.Contains("Retry Exit Codes")); }