public void TestDumpPipLiteWithPassingAndFailingPips()
{
var passingCopyFile = new CopyFile(CreateSourceFile(), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context));
var failingCopyFile = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context));
PipGraphBuilder.AddCopyFile(passingCopyFile);
PipGraphBuilder.AddCopyFile(failingCopyFile);
var schedulerResult = RunScheduler().AssertFailure();
var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips");
var failingPipDumpFile = Path.Combine(logFolder, $"{failingCopyFile.FormattedSemiStableHash}.json");
var passingPipDumpFile = Path.Combine(logFolder, $"{passingCopyFile.FormattedSemiStableHash}.json");
List <Option> options = new List <Option>
{
new Option()
{
Name = "d+"
}
};
RunAnalyzer(schedulerResult, null, options);
Assert.True(Directory.Exists(logFolder));
Assert.True(File.Exists(failingPipDumpFile));
Assert.True(!File.Exists(passingPipDumpFile));
}
public void AffectedSourceInsideSourceSealDirectory()
{
var sourceDirectory = CreateUniqueDirectory(SourceRootPath);
var fileInsideSourceDirectory = CreateSourceFile(sourceDirectory, "file_");
var sourceSealDirectory = CreateSourceSealDirectory(sourceDirectory, SealDirectoryKind.SourceTopDirectoryOnly, "file_*");
PipGraphBuilder.AddSealDirectory(sourceSealDirectory);
var changeAffectedWrittenFile = CreateUniqueObjPath("change");
var pipBuilder = CreatePipBuilder(new Operation[]
{
// Ensure that pip reads the changeAffectedWrittenFile if it exists.
Operation.ReadFile(FileArtifact.CreateSourceFile(changeAffectedWrittenFile), doNotInfer: true),
Operation.ReadFile(fileInsideSourceDirectory, doNotInfer: true),
Operation.WriteFile(CreateOutputFileArtifact()),
});
pipBuilder.AddInputDirectory(sourceSealDirectory.Directory);
pipBuilder.SetChangeAffectedInputListWrittenFile(changeAffectedWrittenFile);
SchedulePipBuilder(pipBuilder);
var inputChangesFile = CreateOutputFileArtifact();
File.WriteAllText(ArtifactToString(inputChangesFile), ArtifactToString(fileInsideSourceDirectory));
Configuration.Schedule.InputChanges = inputChangesFile;
RunScheduler().AssertSuccess();
var actualAffectedInput = File.ReadAllText(changeAffectedWrittenFile.ToString(Context.PathTable));
XAssert.AreEqual(ArtifactToString(fileInsideSourceDirectory), actualAffectedInput);
}
public void TestDumpPipLite()
{
var copyFile = new CopyFile(CreateSourceFile(), CreateOutputFileArtifact(), new List <StringId>().ToReadOnlyArray(), PipProvenance.CreateDummy(Context));
PipGraphBuilder.AddCopyFile(copyFile);
var schedulerResult = RunScheduler().AssertSuccess();
var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips");
var pipDumpFile = Path.Combine(logFolder, $"{copyFile.FormattedSemiStableHash}.json");
List <Option> options = new List <Option>
{
new Option()
{
Name = "o",
Value = schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable)
},
new Option()
{
Name = "p",
Value = copyFile.FormattedSemiStableHash
}
};
RunAnalyzer(schedulerResult, null, options);
Assert.True(Directory.Exists(logFolder));
Assert.True(File.Exists(pipDumpFile));
}
public void TestFailingPipDump(bool failPip)
{
BaseSetup();
FileArtifact sourceArtifact = failPip ? FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)) : CreateSourceFile();
FileArtifact destinationArtifact = CreateOutputFileArtifact();
var copyFile = CreateCopyFile(sourceArtifact, destinationArtifact);
PipGraphBuilder.AddCopyFile(copyFile);
var schedulerResult = RunScheduler();
var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips");
var pipDumpFile = Path.Combine(logFolder, $"{copyFile.FormattedSemiStableHash}.json");
if (failPip)
{
schedulerResult.AssertFailure();
SetExpectedFailures(1, 0); // One error logged for the failing pip
AssertErrorCount();
Assert.True(Directory.Exists(logFolder));
Assert.True(File.Exists(pipDumpFile));
Assert.True(new FileInfo(pipDumpFile).Length > 0); //Ensure that some content was written to the file
}
else
{
schedulerResult.AssertSuccess();
// FailedPips log folder should not have been created when we run successfully
Assert.False(Directory.Exists(logFolder));
Assert.False(File.Exists(pipDumpFile));
}
}
public void FailWhenTwoPipsHaveSameTempDirectory(bool allowDuplicateTemporaryDirectory, int tempArtifactType)
{
Configuration.Engine.AllowDuplicateTemporaryDirectory = allowDuplicateTemporaryDirectory;
Exception caughtExpection = null;
try
{
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut1);
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut2);
PipGraphBuilder.Build();
}
catch (Exception e)
{
caughtExpection = e;
}
if (!allowDuplicateTemporaryDirectory)
{
AssertTrue(caughtExpection != null, "Expected PipGraph Builder Exception, but Graph built successfully");
AssertErrorEventLogged(LogEventId.MultiplePipsUsingSameTemporaryDirectory);
}
else
{
AssertTrue(caughtExpection == null, "Expected PipGraph Builder to pass, but Graph built failed with exception: " + caughtExpection?.Message);
}
}
private void RunAndAssertDumpPip(Pip pip)
{
var graph = PipGraphBuilder.Build();
var success = CreateLogPathAndRun(pip, graph);
AssertCommon(success, pip, graph);
}
public void ConsumeFilterPassingFile(bool topOnly)
{
FileArtifact source;
if (topOnly)
{
source = FileArtifact.CreateSourceFile(SourceRootPath.Combine(Context.PathTable, "file.txt"));
}
else
{
var nestedDir = AbsolutePath.Create(Context.PathTable, Path.Combine(SourceRoot, "nested"));
source = FileArtifact.CreateSourceFile(nestedDir.Combine(Context.PathTable, "file.txt"));
}
WriteSourceFile(source);
var output = CreateOutputFileArtifact(ObjectRoot);
SealDirectory sealedDirectory = CreateSourceSealDirectory(SourceRootPath, topOnly ? SealDirectoryKind.SourceTopDirectoryOnly : SealDirectoryKind.SourceAllDirectories, "*.txt", "*.cs");
DirectoryArtifact dir = PipGraphBuilder.AddSealDirectory(sealedDirectory);
var builder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(source, doNotInfer: true),
Operation.WriteFile(output),
});
builder.AddInputDirectory(dir);
SchedulePipBuilder(builder);
RunScheduler().AssertSuccess();
}
public void ConsumeFilterNotPassingFile()
{
FileArtifact source = FileArtifact.CreateSourceFile(SourceRootPath.Combine(Context.PathTable, "file.txt"));
WriteSourceFile(source);
var output = CreateOutputFileArtifact(ObjectRoot);
SealDirectory sealedDirectory = CreateSourceSealDirectory(SourceRootPath, SealDirectoryKind.SourceTopDirectoryOnly, "*.cs");
DirectoryArtifact dir = PipGraphBuilder.AddSealDirectory(sealedDirectory);
var builder = CreatePipBuilder(new Operation[]
{
Operation.ReadFile(source, doNotInfer: true),
Operation.WriteFile(output),
});
builder.AddInputDirectory(dir);
SchedulePipBuilder(builder);
RunScheduler().AssertFailure();
AssertVerboseEventLogged(ProcessesLogEventId.PipProcessDisallowedFileAccess);
AssertVerboseEventLogged(LogEventId.DependencyViolationMissingSourceDependency);
AssertWarningEventLogged(LogEventId.ProcessNotStoredToCacheDueToFileMonitoringViolations);
AssertErrorEventLogged(LogEventId.FileMonitoringError);
}
public void TestWriteFileDump()
{
FileArtifact outputFile = CreateOutputFileArtifact();
WriteFile pip = CreateWriteFile(outputFile, string.Empty, new[] { "some content" });
PipGraphBuilder.AddWriteFile(pip);
RunAndAssertDumpPip(pip);
}
public void TestModulePipDump()
{
AbsolutePath specPath = CreateUniqueSourcePath();
ModulePip module = ModulePip.CreateForTesting(Context.StringTable, specPath);
PipGraphBuilder.AddModule(module);
RunAndAssertDumpPip(module);
}
/// <summary>
/// Runs the scheduler using the instance member PipGraph and Configuration objects. This will also carry over
/// any state from any previous run such as the cache
/// </summary>
public ScheduleRunResult RunScheduler(SchedulerTestHooks testHooks = null, SchedulerState schedulerState = null, RootFilter filter = null, TempCleaner tempCleaner = null)
{
if (m_graphWasModified || m_lastGraph == null)
{
m_lastGraph = PipGraphBuilder.Build();
XAssert.IsNotNull(m_lastGraph, "Failed to build pip graph");
}
m_graphWasModified = false;
return(RunSchedulerSpecific(m_lastGraph, testHooks, schedulerState, filter, tempCleaner));
}
public void FailWhenCopyFileOutputsToTemp(int tempArtifactType)
{
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut);
var srcFile = CreateSourceFile();
var destFile = CreateOutputFileArtifact(TempRoot);
var copyFilePip = CreateAndScheduleCopyFile(srcFile, destFile);
PipGraphBuilder.Build();
AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath);
}
public void TestSealDirectoryDump()
{
var sealPath = CreateOutputDirectoryArtifact(TemporaryDirectory);
SealDirectory sealDirectory = CreateSealDirectory(sealPath.Path, SealDirectoryKind.Full, scrub: true, new[] { CreateSourceFile(), CreateSourceFile(), CreateSourceFile() });
sealDirectory.SetDirectoryArtifact(sealPath);
PipGraphBuilder.AddSealDirectory(sealDirectory);
RunAndAssertDumpPip(sealDirectory);
}
public void TestCopyFileDump()
{
var sourceArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, GetFullPath("source")));
var destinationArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(Context.PathTable, GetFullPath("destination")));
CopyFile pip = CreateCopyFile(sourceArtifact, destinationArtifact);
PipGraphBuilder.AddCopyFile(pip);
RunAndAssertDumpPip(pip);
}
public void TestProcessDump()
{
var sourceFile = CreateSourceFile();
var outputFile = CreateOutputFileArtifact();
Process process = CreateCmdProcess(dependencies: new[] { sourceFile }, outputs: new[] { outputFile });
PipGraphBuilder.AddProcess(process);
RunAndAssertDumpPip(process);
}
public void FailWhenTempDirectoryContainsTempDirectory(int tempArtifactType)
{
var parentTemp = CreateUniqueDirectory(TempRoot);
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut, parentTemp);
var childTemp = CreateUniqueDirectory(parentTemp);
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut2, childTemp);
PipGraphBuilder.Build();
}
public void FailWhenTempDirectoryContainsAnotherPipsOutputFile(int tempArtifactType)
{
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut);
// Pip attempts to declare an output in a temp directory (that was declared by a different pip)
var pipInvalidOutInTempOps = CreateAndSchedulePipBuilder(new Operation[]
{
Operation.WriteFile(CreateOutputFileArtifact(TempRoot))
}).Process;
PipGraphBuilder.Build();
AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath);
}
public void TestBadPath()
{
FileArtifact outputFile = CreateOutputFileArtifact();
WriteFile pip = CreateWriteFile(outputFile, string.Empty, new[] { "some content" });
PipGraphBuilder.AddWriteFile(pip);
var graph = PipGraphBuilder.Build();
var success = DumpPipLiteAnalysisUtilities.DumpPip(pip, @"X:\not\a\real\path\", Context.PathTable, Context.StringTable, Context.SymbolTable, graph, LoggingContext);
Assert.False(success);
AssertWarningEventLogged(LogEventId.DumpPipLiteUnableToSerializePipDueToBadPath);
}
/// <summary>
/// Serializes and deserializes graph fragments synchronously according to its dependency relation specified by their order in the array.
/// </summary>
/// <param name="fragments">Graph fragments with total order on their dependency relation.</param>
/// <returns>Resulting pip graph.</returns>
private PipGraph SerializeDeserializeFragmentsSynchronously(params TestPipGraphFragment[] fragments)
{
var streams = SerializeFragmentsSynchronously(fragments);
var fragmentManager = new PipGraphFragmentManager(LoggingContext, Context, PipGraphBuilder);
for (int i = 0; i < streams.Length; ++i)
{
XAssert.IsTrue(fragmentManager.AddFragmentFileToGraph(streams[i], fragments[i].ModuleName));
}
DisposeStreams(streams);
return(PipGraphBuilder.Build());
}
public void TestAddSealDirectory()
{
var root = CreateUniqueSourcePath("root");
var seal = CreateSealDirectory(root, SealDirectoryKind.Partial);
PipGraphBuilder.AddSealDirectory(seal);
var builder = new PatchablePipGraph(
oldPipGraph: PipGraphBuilder.DirectedGraph,
oldPipTable: PipTable,
graphBuilder: CreatePipGraphBuilder(),
maxDegreeOfParallelism: Environment.ProcessorCount);
var stats = builder.PartiallyReloadGraph(new HashSet <AbsolutePath>());
Assert.Equal(1, stats.NumPipsReloaded);
}
private Pip[] CreateGraph(int numNodes, string graphAsString)
{
// parse edges
var graph = SimpleGraph.Parse(numNodes, graphAsString);
// for each node create a single output file
var outFiles = Enumerable
.Range(0, numNodes)
.Select(_ => CreateOutputFileArtifact())
.ToArray();
// for each node create a Process pip with a single output file and dependencies according to 'edges'
var processes = Enumerable
.Range(0, numNodes)
.Select(procIdx =>
{
var dependencies = graph
.Edges
.Where(e => e.Dest == procIdx)
.Select(e => outFiles[e.Src])
.ToArray();
var processBuilder = new ProcessBuilder();
var arguments = SchedulerTestBase.CreateCmdArguments(
stringTable: Context.StringTable,
dependencies: dependencies,
outputs: new[] { outFiles[procIdx] });
return(processBuilder
.WithContext(Context)
.WithWorkingDirectory(GetWorkingDirectory())
.WithStandardDirectory(GetStandardDirectory())
.WithExecutable(CmdExecutable)
.WithArguments(arguments)
.WithDependencies(dependencies)
.WithOutputs(outFiles[procIdx])
.WithProvenance(CreateProvenance())
.Build());
})
.ToArray();
// add created processes to PipGraphBuilder
foreach (var proc in processes)
{
PipGraphBuilder.AddProcess(proc);
}
// return created processes
return(processes);
}
public void FailWhenTempDirectoryContainsAnotherPipsOutputDirectory(int tempArtifactType)
{
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut);
// Pip attempts to declare an output directory in a temp directory (that was declared by a different pip)
var builder = CreatePipBuilder(new Operation[] { });
var opaqueDir = Path.Combine(TempRoot, "opaquedir");
var opaqueDirPath = AbsolutePath.Create(Context.PathTable, opaqueDir);
builder.AddOutputDirectory(opaqueDirPath, SealDirectoryKind.Opaque);
SchedulePipBuilder(builder);
PipGraphBuilder.Build();
AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath);
}
public ProcessWithOutputs SchedulePipBuilder(ProcessBuilder builder)
{
AddUntrackedWindowsDirectories(builder);
if (!builder.TryFinish(PipConstructionHelper, out var process, out var outputs))
{
throw new BuildXLTestException("Failed to construct process pip");
}
if (!PipGraphBuilder.AddProcess(process))
{
throw new BuildXLTestException("Failed to add process pip");
}
return(new ProcessWithOutputs(process, outputs));
}
public void FailWhenTempDirectoryContainsSourceFile(int tempArtifactType)
{
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut);
// Declare a dependency on a source file within a different pip's temp directory
var src = CreateSourceFile(TempRootPath);
CreateAndSchedulePipBuilder(new Operation[]
{
Operation.ReadFile(src),
Operation.WriteFile(CreateOutputFileArtifact())
});
PipGraphBuilder.Build();
AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath);
}
public void FailWhenSealedSourceDirectoryIsTempArtifact(int tempArtifactType)
{
var sealDir = SealDirectory(SourceRootPath, SealDirectoryKind.SourceAllDirectories);
var builder = CreatePipBuilder(new Operation[]
{
Operation.WriteFile(CreateOutputFileArtifact())
});
builder.AddInputDirectory(sealDir);
SchedulePipBuilder(builder);
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut, tempRoot: sealDir.Path);
PipGraphBuilder.Build();
AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath);
}
public void FailWhenTempDirectoryContainsPipsOutputFile(int tempArtifactType)
{
var tempOut = CreateOutputFileArtifact(TempRoot);
// Pip attempts to declare an output in its own temp
CreateAndScheduleTempDirProcess(new Operation[]
{
Operation.WriteFile(tempOut),
Operation.WriteFile(CreateOutputFileArtifact())
},
tempArtifactType,
TempRootPath,
tempOut.Path);
PipGraphBuilder.Build();
AssertErrorEventLogged(EventId.InvalidGraphSinceArtifactPathOverlapsTempPath);
}
public void FailWhenTwoPipsHaveSameTempDirectory(int tempArtifactType)
{
Exception exception = null;
try
{
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut1);
CreateAndSchedulePipWithTemp(tempArtifactType, out var tempOut2);
PipGraphBuilder.Build();
}
catch (Exception ex)
{
exception = ex;
}
XAssert.AreNotEqual(null, exception);
}
/// <summary>
/// Serializes and deserializes graph fragments in parallel.
/// </summary>
/// <param name="fragments">Graph fragments.</param>
/// <returns>Resulting pip graph.</returns>
/// <remarks>
/// The graph fragments, <paramref name="fragments"/>, are serialized synchronously, but are deserialized in parallel.
/// For correctness, the graph fragments in <paramref name="fragments"/> are assumed to be independent of each other.
/// </remarks>
private PipGraph SerializeDeserializeFragmentsInParallel(params TestPipGraphFragment[] fragments)
{
var streams = SerializeFragmentsSynchronously(fragments);
var fragmentManager = new PipGraphFragmentManager(LoggingContext, Context, PipGraphBuilder);
Parallel.For(
0,
fragments.Length,
i =>
{
XAssert.IsTrue(fragmentManager.AddFragmentFileToGraph(streams[i], fragments[i].ModuleName));
});
DisposeStreams(streams);
return(PipGraphBuilder.Build());
}
public void TestLogLimit()
{
var failingCopyFile1 = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(Context));
var failingCopyFile2 = new CopyFile(FileArtifact.CreateSourceFile(CreateUniqueSourcePath(SourceRootPrefix)), CreateOutputFileArtifact(), ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(Context));
PipGraphBuilder.AddCopyFile(failingCopyFile1);
PipGraphBuilder.AddCopyFile(failingCopyFile2);
Configuration.Logging.DumpFailedPipsLogLimit = 1;
var schedulerResult = RunScheduler().AssertFailure();
var logFolder = Path.Combine(schedulerResult.Config.Logging.LogsDirectory.ToString(Context.PathTable), "FailedPips");
SetExpectedFailures(2, 0);
AssertErrorCount();
AssertVerboseEventLogged(LogEventId.RuntimeDumpPipLiteLogLimitReached, count: 1);
Assert.True(Directory.GetFiles(logFolder).Length == 1);
}
public void TestProcessDump()
{
var sourceFile = CreateSourceFile();
var outputFile = CreateOutputFileArtifact();
Process process = CreateCmdProcess(dependencies: new[] { sourceFile }, outputs: new[] { outputFile });
PipGraphBuilder.AddProcess(process);
RunAndAssertDumpPip(process);
// Test a few empty list fields to ensure they did not get printed out
var pipWrittenToFile = Encoding.UTF8.GetString(File.ReadAllBytes(GetDumpFilePath(process)));
// This pip does not have any of the following fields set (they will have size 0 lists)
Assert.False(pipWrittenToFile.Contains("Tags"));
Assert.False(pipWrittenToFile.Contains("Environment Variables"));
Assert.False(pipWrittenToFile.Contains("Retry Exit Codes"));
}
