private IEnumerable <Pip> HydrateAllPips(PipTable pipTable)
{
return(pipTable.Keys.Select(pipId => pipTable.HydratePip(pipId, PipQueryContext.Test)).ToList());
}
private Process HydratePip(PipId pipId)
{
return((Process)PipTable.HydratePip(pipId, PipQueryContext.ViewerAnalyzer));
}
public PipQueueTestExecutionEnvironment(
BuildXLContext context,
IConfiguration configuration,
PipTable pipTable,
string tempDirectory,
(string substSource, string substTarget)?subst = default,
public override int Analyze()
{
var rootExpander = new RootExpander(PathTable);
HashSet <ContentHash> hashes = new HashSet <ContentHash>();
hashes.Add(ContentHashingUtilities.ZeroHash);
HashSet <FileArtifact> files = new HashSet <FileArtifact>();
foreach (var root in Roots)
{
rootExpander.Add(AbsolutePath.Create(PathTable, root.Key), root.Value);
}
Func <AbsolutePath, string> expandRoot = absPath => absPath.ToString(PathTable, rootExpander);
var orderedPips = CachedGraph.PipGraph.RetrievePipReferencesOfType(PipType.Process)
.Where(lazyPip => TargetSemiStableHash == null || TargetSemiStableHash == lazyPip.SemiStableHash)
.Select(lazyPip => (Process)lazyPip.HydratePip())
.ToLookup(process => process.Provenance.Token.Path)
.OrderBy(grouping => grouping.Key.ToString(PathTable, rootExpander))
.ToList();
using (var fingerprintStream = File.Create(DumpFilePath, bufferSize: 64 << 10 /* 64 KB */))
using (var hashWriter = new StreamWriter(DumpFilePath + ".hashes.txt"))
{
using (
var fingerprintArchive = CompressFile
? new ZipArchive(fingerprintStream, ZipArchiveMode.Create)
: null)
{
using (
var writer =
XmlWriter.Create(
CompressFile
? fingerprintArchive.CreateEntry("dump.xml", CompressionLevel.Fastest).Open()
: fingerprintStream, new XmlWriterSettings()
{
Indent = true
}))
{
int doneProcesses = 0;
var t = new Timer(
o =>
{
var done = doneProcesses;
Console.WriteLine("Processes Done: {0} of {1}", done, orderedPips.Count);
},
null,
5000,
5000);
try
{
writer.WriteStartElement("ProcessDump");
writer.WriteAttributeString("Count", orderedPips.Count.ToString(CultureInfo.InvariantCulture));
foreach (var specPipGroup in orderedPips)
{
writer.WriteStartElement("SpecFile");
writer.WriteAttributeString("Path", specPipGroup.Key.ToString(PathTable, rootExpander));
foreach (var pip in specPipGroup)
{
doneProcesses++;
writer.WriteStartElement("Process");
writer.WriteAttributeString("Name", pip.Executable.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("CMD", GetArgumentsDataFromProcess(pip).ToString(expandRoot, PathTable.StringTable, PipData.MaxMonikerRenderer));
writer.WriteElementString("Description", pip.GetDescription(PipGraph.Context));
writer.WriteStartElement("EnvironmentVariables");
foreach (var environmentVariable in pip.EnvironmentVariables)
{
writer.WriteStartElement("Environment");
writer.WriteAttributeString("Name", environmentVariable.Name.ToString(PathTable.StringTable));
if (environmentVariable.Value.IsValid)
{
writer.WriteAttributeString("Value", environmentVariable.Value.ToString(expandRoot, PathTable.StringTable, PipData.MaxMonikerRenderer));
}
else
{
writer.WriteAttributeString("Value", "Unset");
}
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("Dependencies");
foreach (var input in pip.Dependencies)
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(input, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString());
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("DirectoryDependencies");
foreach (var input in pip.DirectoryDependencies)
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander));
var kind = PipTable.GetSealDirectoryKind(PipGraph.GetSealedDirectoryNode(input).ToPipId());
writer.WriteAttributeString("Kind", kind.ToString());
// Print directory dependency file details when dumping a specific process
if (TargetSemiStableHash != null && (kind == SealDirectoryKind.Full || kind == SealDirectoryKind.Partial))
{
foreach (var file in PipGraph.ListSealedDirectoryContents(input))
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
writer.WriteEndElement();
}
}
else if (m_contents.TryGetValue(input, out var contents))
{
m_observedInputs.TryGetValue(pip.PipId.Value, out var observedInputs);
foreach (var file in contents)
{
// skip the files that were not accessed
if (observedInputs != null && !observedInputs.Contains(file.Path))
{
continue;
}
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("Outputs");
foreach (var input in pip.FileOutputs)
{
writer.WriteStartElement("Item");
if (input.RewriteCount > 1)
{
writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString());
}
writer.WriteString(input.Path.ToString(PathTable, rootExpander));
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("DirectoryOutputs");
foreach (var output in pip.DirectoryOutputs)
{
writer.WriteStartElement("Directory");
{
writer.WriteAttributeString("Path", output.Path.ToString(PathTable, rootExpander));
if (m_contents.TryGetValue(output, out var contents))
{
writer.WriteStartElement("Contents");
{
foreach (var file in contents)
{
writer.WriteStartElement("Item");
if (file.RewriteCount > 1)
{
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
}
writer.WriteString(file.Path.ToString(PathTable, rootExpander));
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
writer.WriteEndElement();
if (pip.TempDirectory.IsValid)
{
writer.WriteElementString("TempDirectory", pip.TempDirectory.ToString(PathTable, rootExpander));
}
writer.WriteStartElement("AdditionalTempDirectories");
foreach (var item in pip.AdditionalTempDirectories)
{
writer.WriteElementString("Item", item.ToString(PathTable, rootExpander));
}
writer.WriteEndElement();
writer.WriteEndElement(); // Process
}
writer.WriteEndElement(); // SpecFile
}
writer.WriteEndElement(); // ProcessDump
}
finally
{
// kill and wait for the status timer to die...
using (var e = new AutoResetEvent(false))
{
t.Dispose(e);
e.WaitOne();
}
}
}
}
}
return(0);
}
/// <summary>
/// Run the test
/// </summary>
public bool Run(string testFolder, string specFile, string fullIdentifier, string shortName, string lkgFile, params string[] sdksToResolve)
{
Contract.Requires(!string.IsNullOrEmpty(testFolder));
Contract.Requires(!string.IsNullOrEmpty(specFile));
Contract.Requires(sdksToResolve != null);
// Sadly the frontend doesn't use the engine abstractions file api's so we have to materialize stuff on disk for now...
// TODO: Fix this code once the frontend supports a proper virtual FileSystem.
// TODO: Change the package semantics to implicit when we expose a way to evaluate a single value
var testFileName = Path.GetFileName(specFile);
var mainFileName = "testMain.bp";
var testMainFile = Path.Combine(testFolder, mainFileName);
Directory.CreateDirectory(testFolder);
File.WriteAllText(Path.Combine(testFolder, Names.ModuleConfigBm), I($@"module(
{{
name: 'TestPackage',
nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences,
projects: [
f`{mainFileName}`,
f`{testFileName}`,
],
}});"));
File.WriteAllText(testMainFile, I($@"
export const testFolder = d`{Path.GetDirectoryName(specFile).Replace('\\', '/')}`;
@@public
export const main = {fullIdentifier}();"));
File.Copy(specFile, Path.Combine(testFolder, testFileName));
// Create a fake package for Sdk.TestRunner so that you can safely test packages that have the tests embedded in them.
var testRunnerFolder = Path.Combine(testFolder, "Sdk.TestRunner");
Directory.CreateDirectory(testRunnerFolder);
File.WriteAllText(Path.Combine(testRunnerFolder, Names.ModuleConfigBm), I($"module({{\n\tname: 'Sdk.TestRunner',\n}});"));
File.WriteAllText(Path.Combine(testRunnerFolder, "package" + Names.DotDscExtension), I($@"
export interface TestArguments {{
testFiles: File[];
sdkFolders?: (Directory|StaticDirectory)[];
autoFixLkgs?: boolean;
}}
export interface TestResult {{
xmlResults: File;
}}
export function test(args: TestArguments): TestResult {{
Contract.fail(""Can't run a DScript UnitTest inside of a DScript UnitTest"");
}}"));
// Setup Context and configuration
var frontEndContext = FrontEndContext.CreateInstanceForTesting();
var pipContext = new SchedulerContext(CancellationToken.None, frontEndContext.StringTable, frontEndContext.PathTable, frontEndContext.SymbolTable, frontEndContext.QualifierTable);
var pathTable = frontEndContext.PathTable;
var testFolderPath = AbsolutePath.Create(pathTable, testFolder);
var configuration = CreateConfiguration(sdksToResolve.Union(new[] { testRunnerFolder }), pathTable, testFolderPath);
var engineAbstraction = new TestEngineAbstraction(pathTable, frontEndContext.StringTable, testFolderPath, new PassThroughFileSystem(pathTable));
var frontEndStatistics = new FrontEndStatistics();
if (!CreateFactories(
frontEndContext,
engineAbstraction,
frontEndStatistics,
configuration,
out var ambientTesting,
out var workspaceFactory,
out var moduleRegistry,
out var frontEndFactory))
{
return(false);
}
// Set the timeout to a large number to avoid useless performance collections in tests.
using (var performanceCollector = new PerformanceCollector(TimeSpan.FromHours(1)))
using (var frontEndHostController = new FrontEndHostController(
frontEndFactory,
workspaceFactory,
new EvaluationScheduler(1),
moduleRegistry,
frontEndStatistics,
m_tracingLogger,
performanceCollector,
collectMemoryAsSoonAsPossible: true))
{
var frontEndController = (IFrontEndController)frontEndHostController;
frontEndController.InitializeHost(frontEndContext, configuration);
frontEndController.ParseConfig(configuration);
// Populate the graph
using (var pipTable = new PipTable(
pipContext.PathTable,
pipContext.SymbolTable,
initialBufferSize: 16384,
maxDegreeOfParallelism: 1,
debug: true))
{
var mountPathExpander = new MountPathExpander(pathTable);
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "testFolder"), testFolderPath, allowHashing: true, readable: true, writable: false));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "src"), testFolderPath.Combine(pathTable, "src"), allowHashing: true, readable: true, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "out"), testFolderPath.Combine(pathTable, "out"), allowHashing: true, readable: true, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "noRead"), testFolderPath.Combine(pathTable, "noRead"), allowHashing: true, readable: false, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "temp"), engineAbstraction.Layout.TempDirectory, allowHashing: true, readable: true, writable: true));
mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "obj"), engineAbstraction.Layout.ObjectDirectory, allowHashing: true, readable: true, writable: true));
var graph = new PipGraph.Builder(
pipTable,
pipContext,
m_schedulerLogger,
frontEndContext.LoggingContext,
configuration,
mountPathExpander);
using (var cacheLayer = new EngineCache(
new InMemoryArtifactContentCache(),
new InMemoryTwoPhaseFingerprintStore()))
{
var cache = Task.FromResult(Possible.Create(cacheLayer));
try
{
var evaluationFilter = new EvaluationFilter(
pipContext.SymbolTable,
pipContext.PathTable,
new FullSymbol[0],
new[]
{
AbsolutePath.Create(frontEndContext.PathTable, testMainFile),
},
CollectionUtilities.EmptyArray <StringId>());
if (!frontEndController.PopulateGraph(cache, graph, engineAbstraction, evaluationFilter, configuration, configuration.Startup))
{
HandleDiagnostics();
return(false);
}
}
catch (AggregateException e)
{
var baseException = e.GetBaseException();
if (baseException is XunitException)
{
// If it is an XUnit assert, then unwrap the exception and throw that because XUnit other doesn't display the error nicely.
ExceptionDispatchInfo.Capture(baseException).Throw();
}
throw;
}
}
if (!ValidatePips(frontEndContext, graph, testFolderPath, specFile, shortName, lkgFile, ambientTesting.DontValidatePipsEnabled))
{
return(false);
}
}
}
HandleDiagnostics();
return(true);
}
public override int Analyze()
{
Console.WriteLine($"Analyzing");
m_block.Complete();
m_block.CompletionAsync().GetAwaiter().GetResult();
Console.WriteLine($"Writing Graph");
foreach (var pip in PipGraph.RetrieveAllPips())
{
var serializedPip = pip;
var nodeId = pip.PipId.ToNodeId();
bool addEdges = true;
if (pip.PipType == PipType.Process)
{
var entry = GetEntry(pip.PipId);
serializedPip = entry.Process;
addEdges = !entry.AddedEdges;
}
if (addEdges && AddEdgesForPips)
{
using (var scope = m_mutableGraph.AcquireExclusiveIncomingEdgeScope(nodeId))
{
foreach (var edge in DirectedGraph.GetIncomingEdges(nodeId))
{
scope.AddEdge(edge.OtherNode, edge.IsLight);
}
}
}
serializedPip.ResetPipId();
m_pipTable.Add(nodeId.Value, serializedPip);
}
m_mutableGraph.Seal();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.DirectedGraph,
m_mutableGraph.Serialize,
Path.Combine(OutputFilePath, nameof(GraphCacheFile.DirectedGraph)))
.GetAwaiter().GetResult();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.PipTable,
w => m_pipTable.Serialize(w, Environment.ProcessorCount),
Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipTable)))
.GetAwaiter().GetResult();
CachedGraph.Serializer.SerializeToFileAsync(
GraphCacheFile.PipGraphId,
PipGraph.SerializeGraphId,
Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipGraphId)))
.GetAwaiter().GetResult();
Console.WriteLine($"Simulating [Reading]");
var simulator = new BuildSimulatorAnalyzer(Input);
simulator.Increment = SimulatorIncrement ?? simulator.Increment;
simulator.ExecutionData.DirectedGraph = m_mutableGraph;
simulator.OutputDirectory = OutputFilePath;
if (!simulator.ReadExecutionLog())
{
Args.TruncatedXlgWarning();
}
Console.WriteLine($"Simulating [Analyzing]");
simulator.Analyze();
Console.WriteLine($"Blocking Dependency Analysis");
DisplayTable <DepColumn> depTable = new DisplayTable <DepColumn>(" , ");
foreach (var pipId in PipTable.Keys)
{
var pipType = PipTable.GetPipType(pipId);
if (pipType == PipType.Process)
{
var entry = GetEntry(pipId);
(PipId node, ulong cost)maxConsumedDependency = default;
(PipId node, ulong cost)maxDependency = default;
foreach (var dep in entry.PipDependencies)
{
var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.Key.ToNodeId()];
if (!maxDependency.node.IsValid || cost > maxDependency.cost)
{
maxDependency = (dep.Key, cost);
}
if (dep.Value != null && dep.Value.HasFlag(ContentFlag.Consumed))
{
if (!maxConsumedDependency.node.IsValid || cost > maxConsumedDependency.cost)
{
maxConsumedDependency = (dep.Key, cost);
}
}
}
depTable.NextRow();
depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier}");
depTable.Set(DepColumn.MaxConsumedDependency, ToString(maxConsumedDependency.node));
depTable.Set(DepColumn.MaxConsumedDependencyChainCost, maxConsumedDependency.cost.ToMinutes());
depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node));
depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes());
}
else if (pipType == PipType.SealDirectory &&
!PipTable.GetSealDirectoryKind(pipId).IsSourceSeal() &&
!IsSourceOnlySeal(pipId))
{
var seal = (SealDirectory)GetPip(pipId);
var entry = GetEntry(seal.Directory);
(PipId node, ulong cost)maxDependency = default;
foreach (var dep in DirectedGraph.GetIncomingEdges(pipId.ToNodeId()))
{
var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.OtherNode];
if (!maxDependency.node.IsValid || cost > maxDependency.cost)
{
maxDependency = (dep.OtherNode.ToPipId(), cost);
}
}
depTable.NextRow();
depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier} ({entry.FileCount} files)");
depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node));
depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes());
depTable.Set(DepColumn.Directory, seal.DirectoryRoot.ToString(PathTable));
}
}
using (var blockAnalysisWriter = new StreamWriter(Path.Combine(OutputFilePath, "blockAnalysis.txt")))
{
depTable.Write(blockAnalysisWriter);
}
m_writer.Dispose();
Console.WriteLine($"Analyzing complete");
return(0);
}
/// <summary>
/// Generate the output of failure data
/// Failed pips + process chain : This points to the tool that failed.
/// Set of dependencies diff contains: source and output changes. Which of those are directe referenced
///
/// </summary>
/// <param name="isDiff">There is diff data from another log</param>
private void GenerateOutput(bool isDiff)
{
Console.WriteLine("Writing failed pip info to '{0}'.", OutputFilePath);
using (var streamWriter = new StreamWriter(OutputFilePath))
using (JsonWriter writer = new JsonTextWriter(streamWriter))
{
writer.WriteStartObject();
writer.WritePropertyName("Mounts");
{
writer.WriteStartObject();
var mountPathExpander = m_mountPathExpander ?? CachedGraph.MountPathExpander;
foreach (var mountRoot in mountPathExpander.GetAllRoots())
{
var mount = mountPathExpander.GetSemanticPathInfo(mountRoot);
writer.WritePropertyName(mount.RootName.ToString(StringTable));
writer.WriteValue(mountRoot.ToString(PathTable));
}
writer.WriteEndObject();
}
writer.WriteWhitespace(Environment.NewLine);
writer.WritePropertyName("Failures");
{
writer.WriteStartArray();
foreach (var failedPip in m_failedPips)
{
var pip = (Process)m_pipTable.HydratePip(failedPip, PipQueryContext.ViewerAnalyzer);
writer.WriteWhitespace(Environment.NewLine);
writer.WriteStartObject();
// prints the semistable hash.
WritePropertyAndValue(writer, "PipId", ToDisplayString(failedPip));
WritePropertyAndValue(writer, "Working Directory", ToDisplayFilePath(pip.WorkingDirectory));
var provenance = pip.Provenance;
WritePropertyAndValue(
writer,
"Qualifier",
provenance != null ? PipGraph.Context.QualifierTable.GetCanonicalDisplayString(provenance.QualifierId) : string.Empty);
WritePropertyAndValue(
writer,
"OutputValueSymbol",
provenance != null ? provenance.OutputValueSymbol.ToString(SymbolTable) : string.Empty);
var pipPerformance = m_pipPerformance[pip.PipId.Value];
WritePropertyAndValue(writer, "PeakMemoryUsageMb", pipPerformance.PeakMemoryUsageMb.ToString());
WritePropertyAndValue(writer, "NumberOfProcesses", pipPerformance.NumberOfProcesses.ToString());
WritePropertyAndValue(
writer,
"FileMonitoringViolationsNotWhitelisted",
pipPerformance.FileMonitoringViolations.NumFileAccessViolationsNotWhitelisted.ToString());
if (isDiff)
{
WritePropertyAndValue(writer, "NumberOfImputChanges", GetDependencyChangesForPip(pip).ToString());
}
if (m_pipDependencyViolationEventData.TryGetValue(pip.PipId, out var pipDependencyViolationEvent))
{
foreach (var data in pipDependencyViolationEvent)
{
WritePropertyAndValue(writer, "DependencyViolationType", data.ViolationType.ToString());
WritePropertyAndValue(writer, "Path causing violation", ToDisplayFilePath(data.Path));
}
}
WriteProcessChain(writer, pip);
// TODO : Are environment variables usefull for analysis
// : Are are all the failures required or choose count as cmd line arg to take top n
writer.WriteWhitespace(Environment.NewLine);
writer.WriteEndObject();
}
writer.WriteEndArray();
}
writer.WriteWhitespace(Environment.NewLine);
var fileToConsumerMap = m_fileToConsumerMap;
var propertyName = "FileInfo";
if (isDiff)
{
propertyName = "FileInfoDiff";
fileToConsumerMap = m_fileToConsumerMapDiff;
// If any added dependencies add them to the output
if (m_fileToConsumerMapAdded.Count > 0)
{
WriteFileDependencies("FileInfoAdded", m_fileToConsumerMapAdded, writer);
}
// TODO:Add removed dependencies when compared to other log
}
WriteFileDependencies(propertyName, fileToConsumerMap, writer);
writer.WriteWhitespace(Environment.NewLine);
writer.WritePropertyName("PipGraph");
{
writer.WriteStartObject();
m_failedPipsClosure.UnsafeReset();
writer.WritePropertyName("root");
{
writer.WriteStartArray();
foreach (var failedPip in m_failedPips)
{
writer.WriteValue(ToDisplayString(failedPip));
}
writer.WriteEndArray();
}
List <NodeId> dependencyBuffer = new List <NodeId>();
nodeVisitor.VisitTransitiveDependencies(
m_failedPips.Select(p => p.ToNodeId()),
m_failedPipsClosure,
visitNode: node =>
{
dependencyBuffer.Clear();
foreach (var dependencyEdge in DataflowGraph.GetIncomingEdges(node))
{
if (PipTable.GetPipType(dependencyEdge.OtherNode.ToPipId()) != PipType.HashSourceFile)
{
dependencyBuffer.Add(dependencyEdge.OtherNode);
}
}
if (dependencyBuffer.Count != 0)
{
writer.WritePropertyName(ToDisplayString(node.ToPipId()));
{
writer.WriteStartArray();
foreach (var dependencyNode in dependencyBuffer)
{
writer.WriteValue(ToDisplayString(dependencyNode.ToPipId()));
}
writer.WriteEndArray();
}
}
return(true);
});
writer.WriteEndObject();
}
writer.WriteEndObject();
}
}
public PipQueueTestExecutionEnvironment(BuildXLContext context, IConfiguration configuration, PipTable pipTable, IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(configuration != null);
Context = context;
LoggingContext = CreateLoggingContextForTest();
Configuration = configuration;
FileContentTable = FileContentTable.CreateNew();
ContentFingerprinter = new PipContentFingerprinter(
context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
ExtraFingerprintSalts.Default(),
pathExpander: PathExpander);
PipTable = pipTable;
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = IpcFactory.GetProvider();
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
Cache = InMemoryCacheFactory.Create();
LocalDiskContentStore = new LocalDiskContentStore(LoggingContext, context.PathTable, FileContentTable, tracker);
m_sandboxedKextConnection = sandboxedKextConnection;
m_expectedWrittenContent = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_wellKnownFiles = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_producers = new ConcurrentDictionary <FileArtifact, Pip>();
m_filesystemView = new TestPipGraphFilesystemView(Context.PathTable);
var fileSystemView = new FileSystemView(Context.PathTable, m_filesystemView, LocalDiskContentStore);
State = new PipExecutionState(
configuration,
cache: new PipTwoPhaseCache(LoggingContext, Cache, context, PathExpander),
unsafeConfiguration: configuration.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: ContentHashingUtilities.CreateRandom(),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: null,
fileSystemView: fileSystemView,
fileContentManager: new FileContentManager(this, new NullOperationTracker()),
directoryMembershipFinterprinterRuleSet: null);
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
public TestEnv(
string name,
string rootPath,
List <IMount> mounts = null,
PathTable pathTable = null,
Action <CommandLineConfiguration> customizeConfig = null)
{
Contract.Requires(name != null);
Contract.Requires(!string.IsNullOrEmpty(rootPath));
LoggingContext = new LoggingContext("TestLogger." + name);
PathTable = pathTable ?? new PathTable();
PipDataBuilderPool = new ObjectPool <PipDataBuilder>(() => new PipDataBuilder(PathTable.StringTable), _ => { });
// The tests that use TestEnv need to be modernized to take a filesystem
var fileSystem = new PassThroughFileSystem(PathTable);
Context = EngineContext.CreateNew(CancellationToken.None, PathTable, fileSystem);
// Add some well-known paths with fixed casing to the Context.PathTable
AbsolutePath.Create(Context.PathTable, rootPath.ToLowerInvariant());
var root = AbsolutePath.Create(Context.PathTable, rootPath);
var configuration = ConfigHelpers.CreateDefaultForXml(Context.PathTable, root);
configuration.Layout.SourceDirectory = root.Combine(PathTable, PathAtom.Create(PathTable.StringTable, "src")); // These tests have non-standard src folder
configuration.Engine.MaxRelativeOutputDirectoryLength = 260;
configuration.Schedule.EnableLazyOutputMaterialization = false;
configuration.Schedule.UnsafeDisableGraphPostValidation = false;
configuration.Schedule.ComputePipStaticFingerprints = true;
configuration.Sandbox.FileAccessIgnoreCodeCoverage = true;
customizeConfig?.Invoke(configuration);
BuildXLEngine.PopulateFileSystemCapabilities(configuration, configuration, Context.PathTable, LoggingContext);
BuildXLEngine.PopulateLoggingAndLayoutConfiguration(configuration, Context.PathTable, bxlExeLocation: null, inTestMode: true);
BuildXLEngine.PopulateAndValidateConfiguration(configuration, configuration, Context.PathTable, LoggingContext);
Configuration = configuration;
var mountsTable = MountsTable.CreateAndRegister(LoggingContext, Context, Configuration, null);
if (mounts != null)
{
foreach (var mount in mounts)
{
mountsTable.AddResolvedMount(mount);
}
}
AbsolutePath specFile = SourceRoot.CreateRelative(Context.PathTable, "TestSpecFile.dsc");
var graph = TestSchedulerFactory.CreateEmptyPipGraph(Context, configuration, mountsTable.MountPathExpander);
PipTable = graph.PipTable;
PipGraph = graph;
var locationData = new LocationData(specFile, 0, 0);
var modulePip = ModulePip.CreateForTesting(Context.StringTable, specFile);
PipGraph.AddModule(modulePip);
PipGraph.AddSpecFile(new SpecFilePip(FileArtifact.CreateSourceFile(specFile), locationData, modulePip.Module));
PipConstructionHelper = PipConstructionHelper.CreateForTesting(
Context,
ObjectRoot,
redirectedRoot: Configuration.Layout.RedirectedDirectory,
pipGraph: PipGraph,
moduleName: modulePip.Identity.ToString(Context.StringTable),
symbol: name,
specPath: specFile);
Paths = new Paths(PathTable);
mountsTable.CompleteInitialization();
}
/// <summary>
/// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>.
/// </summary>
/// <returns>
/// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to
/// a <see cref="Tracing.FingerprintStore"/> at the provided directory;
/// otherwise, null.
/// </returns>
public static FingerprintStoreExecutionLogTarget Create(
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
LoggingContext loggingContext,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
CounterCollection <FingerprintStoreCounters> counters,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null,
FingerprintStoreTestHooks testHooks = null)
{
var fingerprintStorePathString = configuration.Layout.FingerprintStoreDirectory.ToString(context.PathTable);
var cacheLookupFingerprintStorePathString = configuration.Logging.CacheLookupFingerprintStoreLogDirectory.ToString(context.PathTable);
try
{
FileUtilities.CreateDirectoryWithRetry(fingerprintStorePathString);
}
catch (BuildXLException ex)
{
Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message);
throw new BuildXLException("Unable to create fingerprint store directory: ", ex);
}
var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0);
var possibleExecutionStore = FingerprintStore.Open(
fingerprintStorePathString,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
counters: counters,
testHooks: testHooks);
Possible <FingerprintStore> possibleCacheLookupStore = new Failure <string>("No attempt to create a cache lookup fingerprint store yet.");
if (configuration.Logging.FingerprintStoreMode != FingerprintStoreMode.ExecutionFingerprintsOnly)
{
try
{
FileUtilities.CreateDirectoryWithRetry(cacheLookupFingerprintStorePathString);
}
catch (BuildXLException ex)
{
Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message);
throw new BuildXLException("Unable to create fingerprint store directory: ", ex);
}
possibleCacheLookupStore = FingerprintStore.Open(
cacheLookupFingerprintStorePathString,
maxEntryAge: maxEntryAge,
mode: configuration.Logging.FingerprintStoreMode,
loggingContext: loggingContext,
counters: counters,
testHooks: testHooks);
}
if (possibleExecutionStore.Succeeded &&
(possibleCacheLookupStore.Succeeded || configuration.Logging.FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly))
{
return(new FingerprintStoreExecutionLogTarget(
loggingContext,
context,
pipTable,
pipContentFingerprinter,
possibleExecutionStore.Result,
possibleCacheLookupStore.Succeeded ? possibleCacheLookupStore.Result : null,
configuration,
cache,
graph,
counters,
runnablePipPerformance));
}
else
{
if (!possibleExecutionStore.Succeeded)
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleExecutionStore.Failure.DescribeIncludingInnerFailures());
}
if (!possibleCacheLookupStore.Succeeded)
{
Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleCacheLookupStore.Failure.DescribeIncludingInnerFailures());
}
}
return(null);
}
private IEnumerable <PipReference> AllPipIds(PipTable pipTable)
{
return(pipTable.StableKeys
.Where(pipId => pipTable.GetPipType(pipId) != PipType.HashSourceFile)
.Select(pipId => new PipReference(pipTable, pipId, PipQueryContext.Explorer)));
}
public override int Analyze()
{
Console.WriteLine("Writing failed pip info to '{0}'.", OutputFilePath);
using (var streamWriter = new StreamWriter(OutputFilePath))
using (JsonWriter writer = new JsonTextWriter(streamWriter))
{
writer.WriteStartObject();
writer.WritePropertyName("Mounts");
{
writer.WriteStartObject();
var mountPathExpander = m_mountPathExpander ?? CachedGraph.MountPathExpander;
foreach (var mountRoot in mountPathExpander.GetAllRoots())
{
var mount = mountPathExpander.GetSemanticPathInfo(mountRoot);
writer.WritePropertyName(mount.RootName.ToString(StringTable));
writer.WriteValue(mountRoot.ToString(PathTable));
}
writer.WriteEndObject();
}
writer.WritePropertyName("FileInfo");
{
writer.WriteStartObject();
foreach (var fileEntry in m_fileToConsumerMap)
{
var file = fileEntry.Key;
var consumers = fileEntry.Value;
var path = ToDisplayFilePath(file);
if (path != null)
{
writer.WritePropertyName(path, true);
{
writer.WriteStartArray();
foreach (var consumer in consumers)
{
writer.WriteValue(ToDisplayString(consumer));
}
writer.WriteEndArray();
}
}
}
writer.WriteEndObject();
}
writer.WritePropertyName("PipGraph");
{
writer.WriteStartObject();
m_failedPipsClosure.UnsafeReset();
writer.WritePropertyName("root");
{
writer.WriteStartArray();
foreach (var failedPip in m_failedPips)
{
writer.WriteValue(ToDisplayString(failedPip));
}
writer.WriteEndArray();
}
List <NodeId> dependencyBuffer = new List <NodeId>();
nodeVisitor.VisitTransitiveDependencies(m_failedPips.Select(p => p.ToNodeId()), m_failedPipsClosure, visitNode: node =>
{
dependencyBuffer.Clear();
foreach (var dependencyEdge in DataflowGraph.GetIncomingEdges(node))
{
if (PipTable.GetPipType(dependencyEdge.OtherNode.ToPipId()) != PipType.HashSourceFile)
{
dependencyBuffer.Add(dependencyEdge.OtherNode);
}
}
if (dependencyBuffer.Count != 0)
{
writer.WritePropertyName(ToDisplayString(node.ToPipId()));
{
writer.WriteStartArray();
foreach (var dependencyNode in dependencyBuffer)
{
writer.WriteValue(ToDisplayString(dependencyNode.ToPipId()));
}
writer.WriteEndArray();
}
}
return(true);
});
writer.WriteEndObject();
}
writer.WriteEndObject();
}
return(0);
}
/// <nodoc />
public static Xldb.Proto.PipGraph ToPipGraph(this PipGraph pipGraph, PathTable pathTable, PipTable pipTable, NameExpander nameExpander)
{
var xldbPipGraph = new Xldb.Proto.PipGraph()
{
GraphId = pipGraph.GraphId.ToString(),
SemistableFingerprint = new ContentFingerprint()
{
Hash = pipGraph.SemistableFingerprint.Hash.ToFingerprint()
},
NodeCount = pipTable.StableKeys.Count,
MaxAbsolutePathIndex = pipGraph.MaxAbsolutePathIndex,
FileCount = pipGraph.FileCount,
ContentCount = pipGraph.ContentCount,
ArtifactContentCount = pipGraph.ArtifactContentCount,
ApiServerMoniker = pipGraph.ApiServerMoniker.ToString(pathTable),
};
xldbPipGraph.AllSealDirectoriesAndProducers.AddRange(pipGraph.AllSealDirectoriesAndProducers.Select(kvp => new DirectoryArtifactMap()
{
Artifact = kvp.Key.ToDirectoryArtifact(pathTable, nameExpander),
PipId = kvp.Value.Value
}));
xldbPipGraph.OutputDirectoryExclusions.AddRange(
pipGraph.OutputDirectoryExclusions.UnsafeGetList().Select(p => p.ToAbsolutePath(pathTable, nameExpander)));
xldbPipGraph.StableKeys.AddRange(pipTable.StableKeys.Select(stableKey => stableKey.Value));
foreach (var kvp in pipGraph.Modules)
{
xldbPipGraph.Modules.Add(kvp.Key.Value.ToString(pathTable), kvp.Value.Value);
}
return(xldbPipGraph);
}
/// <summary>
/// Analyzes cache misses.
/// </summary>
public override int Analyze()
{
Console.WriteLine("Starting fingerprint store analysis at: " + DateTime.Now.ToString());
if (!m_newReader.TryGetCacheMissList(out var cacheMissList))
{
WriteLine("Could not retrieve cache miss list for analysis. There may have been a failure during the build.");
return(0);
}
Console.WriteLine("Finished getting list of pips that had a cache miss at: " + DateTime.Now.ToString());
if (m_oldReader.StoreVersion != m_newReader.StoreVersion)
{
WriteLine($"WARNING: Format version numbers of the fingerprint store do not match. Old: {m_oldReader.StoreVersion}, New: {m_newReader.StoreVersion}.");
}
if (SemiStableHashToRun != -1)
{
var firstMiss = cacheMissList.FirstOrDefault(x => PipTable.GetPipSemiStableHash(x.PipId) == SemiStableHashToRun);
if (firstMiss.CacheMissType == PipCacheMissType.Invalid)
{
foreach (var pipId in PipTable.StableKeys)
{
var possibleMatch = PipTable.GetPipSemiStableHash(pipId);
if (possibleMatch == SemiStableHashToRun)
{
firstMiss = new PipCacheMissInfo(pipId, PipCacheMissType.Hit);
}
}
}
Console.WriteLine("Analyzing single pip starting at: " + DateTime.Now.ToString());
AnalyzePip(firstMiss, HydratePip(firstMiss.PipId), m_writer);
}
else
{
var frontierCacheMissList = new List <PipCacheMissInfo>();
foreach (var miss in cacheMissList)
{
if (m_model.HasChangedDependencies(miss.PipId) && !AllPips)
{
continue;
}
frontierCacheMissList.Add(miss);
m_model.MarkChanged(miss.PipId);
}
Console.WriteLine("Finding frontier pips done at " + DateTime.Now.ToString());
int i = 0;
foreach (var miss in frontierCacheMissList)
{
if (i % 10 == 0)
{
Console.WriteLine("Done " + i + " of " + cacheMissList.Count);
}
var pip = HydratePip(miss.PipId);
WriteLine($"================== Analyzing pip ========================");
AnalyzePip(miss, pip, m_writer);
WriteLine("================== Complete pip ========================");
WriteLine();
i++;
}
}
return(0);
}
private string ToDisplayString(PipId pipId)
{
return("pip" + PipTable.GetPipSemiStableHash(pipId).ToString("X16", CultureInfo.InvariantCulture));
}
/// <summary>
/// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified.
/// </summary>
public DummyPipExecutionEnvironment(
LoggingContext loggingContext,
PipExecutionContext context,
IConfiguration config,
FileContentTable fileContentTable = null,
EngineCache pipCache = null,
SemanticPathExpander semanticPathExpander = null,
PipContentFingerprinter.PipDataLookup pipDataLookup = null,
FileAccessWhitelist fileAccessWhitelist = null,
bool allowUnspecifiedSealedDirectories = false,
PipTable pipTable = null,
IIpcProvider ipcProvider = null,
IKextConnection sandboxedKextConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(config != null);
LoggingContext = loggingContext;
Context = context;
// Ensure paths visible when debugging
PathTable.DebugPathTable = Context.PathTable;
Configuration = config;
PipTable = pipTable;
PathExpander = semanticPathExpander ?? SemanticPathExpander.Default;
ContentFingerprinter = new PipContentFingerprinter(
Context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null),
pathExpander: PathExpander,
pipDataLookup: pipDataLookup);
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = ipcProvider ?? IpcFactory.GetProvider();
FileContentTable = fileContentTable ?? FileContentTable.CreateNew();
Cache = pipCache;
FileAccessWhitelist = fileAccessWhitelist;
m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories;
m_sandboxedKextConnection = sandboxedKextConnection;
if (Cache == null)
{
Cache = InMemoryCacheFactory.Create();
}
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker);
PipGraphView = new TestPipGraphFilesystemView(Context.PathTable);
m_operationTracker = new OperationTracker(loggingContext);
var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore);
var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed;
if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled)
{
preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString());
}
State = new PipExecutionState(
config,
cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: ExecutionLogRecorder,
fileSystemView: fileSystemView,
fileContentManager: GetFileContentManager(),
directoryMembershipFinterprinterRuleSet: null,
unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: preserveOutputsSalt,
serviceManager: new DummyServiceManager());
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
public RequiredDependencyAnalyzer(AnalysisInput input)
: base(input)
{
m_pool = new ObjectPool <WorkerAnalyzer>(() => CreateAnalyzer(), w => { });
m_pipTable = EngineSchedule.CreateEmptyPipTable(input.CachedGraph.Context);
}
public async Task Stress()
{
const int N = 5;
const int M = N * N;
var context = BuildXLContext.CreateInstanceForTesting();
var loggingContext = CreateLoggingContextForTest();
var pathTable = context.PathTable;
using (var tempFiles = new TempFileStorage(canGetFileNames: true))
{
var config = ConfigHelpers.CreateDefault(pathTable, tempFiles.GetUniqueFileName(), tempFiles);
using (var pipTable = new PipTable(
context.PathTable,
context.SymbolTable,
initialBufferSize: 1024,
maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3,
debug: false))
{
var executionEnvironment = new PipQueueTestExecutionEnvironment(context, config, pipTable, GetSandboxedKextConnection());
Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) =>
{
PipResult result;
var operationTracker = new OperationTracker(runnablePip.LoggingContext);
var pip = runnablePip.Pip;
using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext))
{
result = await TestPipExecutor.ExecuteAsync(operationContext, executionEnvironment, pip);
}
executionEnvironment.MarkExecuted(pip);
return(result);
};
string executable = CmdHelper.OsShellExe;
FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable));
// This is the only file artifact we reference without a producer. Rather than scheduling a hashing pip, let's just invent one (so fingerprinting can succeed).
executionEnvironment.AddWellKnownFile(executableArtifact, WellKnownContentHashes.UntrackedFile);
using (var phase1PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 1: create some files
var baseFileArtifacts = new List <FileArtifact>();
for (int i = 0; i < N; i++)
{
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
baseFileArtifacts.Add(destinationArtifact);
PipData contents = PipDataBuilder.CreatePipData(
context.StringTable,
" ",
PipDataFragmentEscaping.CRuntimeArgumentRules,
i.ToString(CultureInfo.InvariantCulture));
var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
var pipId = pipTable.Add((uint)(i + 1), writeFile);
var contentHash = ContentHashingUtilities.HashString(contents.ToString(pathTable));
executionEnvironment.AddExpectedWrite(writeFile, destinationArtifact, contentHash);
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactory, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(DispatcherKind.IO);
phase1PipQueue.Enqueue(runnable);
}
phase1PipQueue.SetAsFinalized();
phase1PipQueue.DrainQueues();
await Task.WhenAll(
Enumerable.Range(0, 2).Select(
async range =>
{
using (var phase2PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule))
{
// phase 2: do some more with those files
var pips = new ConcurrentDictionary <PipId, Tuple <string, int> >();
var checkerTasks = new ConcurrentQueue <Task>();
Action <PipId, Task <PipResult> > callback =
(id, task) =>
{
XAssert.IsTrue(task.Status == TaskStatus.RanToCompletion);
XAssert.IsFalse(task.Result.Status.IndicatesFailure());
Tuple <string, int> t;
if (!pips.TryRemove(id, out t))
{
XAssert.Fail();
}
checkerTasks.Enqueue(
Task.Run(
() =>
{
string actual = File.ReadAllText(t.Item1).Trim();
// TODO: Make this async
XAssert.AreEqual(actual, t.Item2.ToString());
}));
};
var r = new Random(0);
for (int i = 0; i < M; i++)
{
int sourceIndex = r.Next(baseFileArtifacts.Count);
FileArtifact sourceArtifact = baseFileArtifacts[sourceIndex];
string destination = tempFiles.GetUniqueFileName();
AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination);
FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion();
Pip pip;
DispatcherKind queueKind;
switch (r.Next(2))
{
case 0:
pip = new CopyFile(sourceArtifact, destinationArtifact, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context));
queueKind = DispatcherKind.IO;
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
case 1:
string workingDirectory =
OperatingSystemHelper.IsUnixOS ? "/tmp" :
Environment.GetFolderPath(Environment.SpecialFolder.Windows);
AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory);
var pipData = OperatingSystemHelper.IsUnixOS ?
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-c", "'", "cp", sourceArtifact, destinationArtifact, "'") :
PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "/d", "/c", "copy", "/B", sourceArtifact, destinationArtifact);
queueKind = DispatcherKind.CPU;
pip = new Process(
executableArtifact,
workingDirectoryAbsolutePath,
pipData,
FileArtifact.Invalid,
PipData.Invalid,
ReadOnlyArray <EnvironmentVariable> .Empty,
FileArtifact.Invalid,
FileArtifact.Invalid,
FileArtifact.Invalid,
tempFiles.GetUniqueDirectory(pathTable),
null,
null,
ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact, sourceArtifact),
ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(destinationArtifact.WithAttributes()),
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <DirectoryArtifact> .Empty,
ReadOnlyArray <PipId> .Empty,
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)),
ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)),
ReadOnlyArray <StringId> .Empty,
ReadOnlyArray <int> .Empty,
ReadOnlyArray <ProcessSemaphoreInfo> .Empty,
provenance: PipProvenance.CreateDummy(context),
toolDescription: StringId.Invalid,
additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty);
executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact));
break;
default:
Contract.Assert(false);
continue;
}
var pipId = pipTable.Add((uint)((range *M) + N + i + 1), pip);
Func <RunnablePip, Task> taskFactoryWithCallback = async(runnablePip) =>
{
var task = taskFactory(runnablePip);
var pipResult = await task;
callback(pipId, task);
};
var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactoryWithCallback, 0);
runnable.Start(new OperationTracker(loggingContext), loggingContext);
runnable.SetDispatcherKind(queueKind);
phase2PipQueue.Enqueue(runnable);
if (!pips.TryAdd(pipId, Tuple.Create(destination, sourceIndex)))
{
Contract.Assert(false);
}
}
phase2PipQueue.SetAsFinalized();
phase2PipQueue.DrainQueues();
XAssert.AreEqual(0, pips.Count);
await Task.WhenAll(checkerTasks);
}
}));
}
}
}
}
public PipsAnalyzer(AnalysisInput input)
: base(input)
{
m_pipTable = input.CachedGraph.PipTable;
}
