private IEnumerable <Pip> HydrateAllPips(PipTable pipTable) { return(pipTable.Keys.Select(pipId => pipTable.HydratePip(pipId, PipQueryContext.Test)).ToList()); }
private Process HydratePip(PipId pipId) { return((Process)PipTable.HydratePip(pipId, PipQueryContext.ViewerAnalyzer)); }
public PipQueueTestExecutionEnvironment( BuildXLContext context, IConfiguration configuration, PipTable pipTable, string tempDirectory, (string substSource, string substTarget)?subst = default,
public override int Analyze() { var rootExpander = new RootExpander(PathTable); HashSet <ContentHash> hashes = new HashSet <ContentHash>(); hashes.Add(ContentHashingUtilities.ZeroHash); HashSet <FileArtifact> files = new HashSet <FileArtifact>(); foreach (var root in Roots) { rootExpander.Add(AbsolutePath.Create(PathTable, root.Key), root.Value); } Func <AbsolutePath, string> expandRoot = absPath => absPath.ToString(PathTable, rootExpander); var orderedPips = CachedGraph.PipGraph.RetrievePipReferencesOfType(PipType.Process) .Where(lazyPip => TargetSemiStableHash == null || TargetSemiStableHash == lazyPip.SemiStableHash) .Select(lazyPip => (Process)lazyPip.HydratePip()) .ToLookup(process => process.Provenance.Token.Path) .OrderBy(grouping => grouping.Key.ToString(PathTable, rootExpander)) .ToList(); using (var fingerprintStream = File.Create(DumpFilePath, bufferSize: 64 << 10 /* 64 KB */)) using (var hashWriter = new StreamWriter(DumpFilePath + ".hashes.txt")) { using ( var fingerprintArchive = CompressFile ? new ZipArchive(fingerprintStream, ZipArchiveMode.Create) : null) { using ( var writer = XmlWriter.Create( CompressFile ? fingerprintArchive.CreateEntry("dump.xml", CompressionLevel.Fastest).Open() : fingerprintStream, new XmlWriterSettings() { Indent = true })) { int doneProcesses = 0; var t = new Timer( o => { var done = doneProcesses; Console.WriteLine("Processes Done: {0} of {1}", done, orderedPips.Count); }, null, 5000, 5000); try { writer.WriteStartElement("ProcessDump"); writer.WriteAttributeString("Count", orderedPips.Count.ToString(CultureInfo.InvariantCulture)); foreach (var specPipGroup in orderedPips) { writer.WriteStartElement("SpecFile"); writer.WriteAttributeString("Path", specPipGroup.Key.ToString(PathTable, rootExpander)); foreach (var pip in specPipGroup) { doneProcesses++; writer.WriteStartElement("Process"); writer.WriteAttributeString("Name", pip.Executable.Path.ToString(PathTable, rootExpander)); writer.WriteAttributeString("CMD", GetArgumentsDataFromProcess(pip).ToString(expandRoot, PathTable.StringTable, PipData.MaxMonikerRenderer)); writer.WriteElementString("Description", pip.GetDescription(PipGraph.Context)); writer.WriteStartElement("EnvironmentVariables"); foreach (var environmentVariable in pip.EnvironmentVariables) { writer.WriteStartElement("Environment"); writer.WriteAttributeString("Name", environmentVariable.Name.ToString(PathTable.StringTable)); if (environmentVariable.Value.IsValid) { writer.WriteAttributeString("Value", environmentVariable.Value.ToString(expandRoot, PathTable.StringTable, PipData.MaxMonikerRenderer)); } else { writer.WriteAttributeString("Value", "Unset"); } writer.WriteEndElement(); } writer.WriteEndElement(); writer.WriteStartElement("Dependencies"); foreach (var input in pip.Dependencies) { writer.WriteStartElement("Item"); writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander)); writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(input, ContentHashingUtilities.ZeroHash).Item.Value.ToString()); writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString()); writer.WriteEndElement(); } writer.WriteEndElement(); writer.WriteStartElement("DirectoryDependencies"); foreach (var input in pip.DirectoryDependencies) { writer.WriteStartElement("Item"); writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander)); var kind = PipTable.GetSealDirectoryKind(PipGraph.GetSealedDirectoryNode(input).ToPipId()); writer.WriteAttributeString("Kind", kind.ToString()); // Print directory dependency file details when dumping a specific process if (TargetSemiStableHash != null && (kind == SealDirectoryKind.Full || kind == SealDirectoryKind.Partial)) { foreach (var file in PipGraph.ListSealedDirectoryContents(input)) { writer.WriteStartElement("Item"); writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander)); writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString()); writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString()); writer.WriteEndElement(); } } else if (m_contents.TryGetValue(input, out var contents)) { m_observedInputs.TryGetValue(pip.PipId.Value, out var observedInputs); foreach (var file in contents) { // skip the files that were not accessed if (observedInputs != null && !observedInputs.Contains(file.Path)) { continue; } writer.WriteStartElement("Item"); writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander)); writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString()); writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString()); writer.WriteEndElement(); } } writer.WriteEndElement(); } writer.WriteEndElement(); writer.WriteStartElement("Outputs"); foreach (var input in pip.FileOutputs) { writer.WriteStartElement("Item"); if (input.RewriteCount > 1) { writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString()); } writer.WriteString(input.Path.ToString(PathTable, rootExpander)); writer.WriteEndElement(); } writer.WriteEndElement(); writer.WriteStartElement("DirectoryOutputs"); foreach (var output in pip.DirectoryOutputs) { writer.WriteStartElement("Directory"); { writer.WriteAttributeString("Path", output.Path.ToString(PathTable, rootExpander)); if (m_contents.TryGetValue(output, out var contents)) { writer.WriteStartElement("Contents"); { foreach (var file in contents) { writer.WriteStartElement("Item"); if (file.RewriteCount > 1) { writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString()); } writer.WriteString(file.Path.ToString(PathTable, rootExpander)); writer.WriteEndElement(); } } writer.WriteEndElement(); } } writer.WriteEndElement(); } writer.WriteEndElement(); if (pip.TempDirectory.IsValid) { writer.WriteElementString("TempDirectory", pip.TempDirectory.ToString(PathTable, rootExpander)); } writer.WriteStartElement("AdditionalTempDirectories"); foreach (var item in pip.AdditionalTempDirectories) { writer.WriteElementString("Item", item.ToString(PathTable, rootExpander)); } writer.WriteEndElement(); writer.WriteEndElement(); // Process } writer.WriteEndElement(); // SpecFile } writer.WriteEndElement(); // ProcessDump } finally { // kill and wait for the status timer to die... using (var e = new AutoResetEvent(false)) { t.Dispose(e); e.WaitOne(); } } } } } return(0); }
/// <summary> /// Run the test /// </summary> public bool Run(string testFolder, string specFile, string fullIdentifier, string shortName, string lkgFile, params string[] sdksToResolve) { Contract.Requires(!string.IsNullOrEmpty(testFolder)); Contract.Requires(!string.IsNullOrEmpty(specFile)); Contract.Requires(sdksToResolve != null); // Sadly the frontend doesn't use the engine abstractions file api's so we have to materialize stuff on disk for now... // TODO: Fix this code once the frontend supports a proper virtual FileSystem. // TODO: Change the package semantics to implicit when we expose a way to evaluate a single value var testFileName = Path.GetFileName(specFile); var mainFileName = "testMain.bp"; var testMainFile = Path.Combine(testFolder, mainFileName); Directory.CreateDirectory(testFolder); File.WriteAllText(Path.Combine(testFolder, Names.ModuleConfigBm), I($@"module( {{ name: 'TestPackage', nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences, projects: [ f`{mainFileName}`, f`{testFileName}`, ], }});")); File.WriteAllText(testMainFile, I($@" export const testFolder = d`{Path.GetDirectoryName(specFile).Replace('\\', '/')}`; @@public export const main = {fullIdentifier}();")); File.Copy(specFile, Path.Combine(testFolder, testFileName)); // Create a fake package for Sdk.TestRunner so that you can safely test packages that have the tests embedded in them. var testRunnerFolder = Path.Combine(testFolder, "Sdk.TestRunner"); Directory.CreateDirectory(testRunnerFolder); File.WriteAllText(Path.Combine(testRunnerFolder, Names.ModuleConfigBm), I($"module({{\n\tname: 'Sdk.TestRunner',\n}});")); File.WriteAllText(Path.Combine(testRunnerFolder, "package" + Names.DotDscExtension), I($@" export interface TestArguments {{ testFiles: File[]; sdkFolders?: (Directory|StaticDirectory)[]; autoFixLkgs?: boolean; }} export interface TestResult {{ xmlResults: File; }} export function test(args: TestArguments): TestResult {{ Contract.fail(""Can't run a DScript UnitTest inside of a DScript UnitTest""); }}")); // Setup Context and configuration var frontEndContext = FrontEndContext.CreateInstanceForTesting(); var pipContext = new SchedulerContext(CancellationToken.None, frontEndContext.StringTable, frontEndContext.PathTable, frontEndContext.SymbolTable, frontEndContext.QualifierTable); var pathTable = frontEndContext.PathTable; var testFolderPath = AbsolutePath.Create(pathTable, testFolder); var configuration = CreateConfiguration(sdksToResolve.Union(new[] { testRunnerFolder }), pathTable, testFolderPath); var engineAbstraction = new TestEngineAbstraction(pathTable, frontEndContext.StringTable, testFolderPath, new PassThroughFileSystem(pathTable)); var frontEndStatistics = new FrontEndStatistics(); if (!CreateFactories( frontEndContext, engineAbstraction, frontEndStatistics, configuration, out var ambientTesting, out var workspaceFactory, out var moduleRegistry, out var frontEndFactory)) { return(false); } // Set the timeout to a large number to avoid useless performance collections in tests. using (var performanceCollector = new PerformanceCollector(TimeSpan.FromHours(1))) using (var frontEndHostController = new FrontEndHostController( frontEndFactory, workspaceFactory, new EvaluationScheduler(1), moduleRegistry, frontEndStatistics, m_tracingLogger, performanceCollector, collectMemoryAsSoonAsPossible: true)) { var frontEndController = (IFrontEndController)frontEndHostController; frontEndController.InitializeHost(frontEndContext, configuration); frontEndController.ParseConfig(configuration); // Populate the graph using (var pipTable = new PipTable( pipContext.PathTable, pipContext.SymbolTable, initialBufferSize: 16384, maxDegreeOfParallelism: 1, debug: true)) { var mountPathExpander = new MountPathExpander(pathTable); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "testFolder"), testFolderPath, allowHashing: true, readable: true, writable: false)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "src"), testFolderPath.Combine(pathTable, "src"), allowHashing: true, readable: true, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "out"), testFolderPath.Combine(pathTable, "out"), allowHashing: true, readable: true, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "noRead"), testFolderPath.Combine(pathTable, "noRead"), allowHashing: true, readable: false, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "temp"), engineAbstraction.Layout.TempDirectory, allowHashing: true, readable: true, writable: true)); mountPathExpander.Add(pathTable, new SemanticPathInfo(PathAtom.Create(frontEndContext.StringTable, "obj"), engineAbstraction.Layout.ObjectDirectory, allowHashing: true, readable: true, writable: true)); var graph = new PipGraph.Builder( pipTable, pipContext, m_schedulerLogger, frontEndContext.LoggingContext, configuration, mountPathExpander); using (var cacheLayer = new EngineCache( new InMemoryArtifactContentCache(), new InMemoryTwoPhaseFingerprintStore())) { var cache = Task.FromResult(Possible.Create(cacheLayer)); try { var evaluationFilter = new EvaluationFilter( pipContext.SymbolTable, pipContext.PathTable, new FullSymbol[0], new[] { AbsolutePath.Create(frontEndContext.PathTable, testMainFile), }, CollectionUtilities.EmptyArray <StringId>()); if (!frontEndController.PopulateGraph(cache, graph, engineAbstraction, evaluationFilter, configuration, configuration.Startup)) { HandleDiagnostics(); return(false); } } catch (AggregateException e) { var baseException = e.GetBaseException(); if (baseException is XunitException) { // If it is an XUnit assert, then unwrap the exception and throw that because XUnit other doesn't display the error nicely. ExceptionDispatchInfo.Capture(baseException).Throw(); } throw; } } if (!ValidatePips(frontEndContext, graph, testFolderPath, specFile, shortName, lkgFile, ambientTesting.DontValidatePipsEnabled)) { return(false); } } } HandleDiagnostics(); return(true); }
public override int Analyze() { Console.WriteLine($"Analyzing"); m_block.Complete(); m_block.CompletionAsync().GetAwaiter().GetResult(); Console.WriteLine($"Writing Graph"); foreach (var pip in PipGraph.RetrieveAllPips()) { var serializedPip = pip; var nodeId = pip.PipId.ToNodeId(); bool addEdges = true; if (pip.PipType == PipType.Process) { var entry = GetEntry(pip.PipId); serializedPip = entry.Process; addEdges = !entry.AddedEdges; } if (addEdges && AddEdgesForPips) { using (var scope = m_mutableGraph.AcquireExclusiveIncomingEdgeScope(nodeId)) { foreach (var edge in DirectedGraph.GetIncomingEdges(nodeId)) { scope.AddEdge(edge.OtherNode, edge.IsLight); } } } serializedPip.ResetPipId(); m_pipTable.Add(nodeId.Value, serializedPip); } m_mutableGraph.Seal(); CachedGraph.Serializer.SerializeToFileAsync( GraphCacheFile.DirectedGraph, m_mutableGraph.Serialize, Path.Combine(OutputFilePath, nameof(GraphCacheFile.DirectedGraph))) .GetAwaiter().GetResult(); CachedGraph.Serializer.SerializeToFileAsync( GraphCacheFile.PipTable, w => m_pipTable.Serialize(w, Environment.ProcessorCount), Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipTable))) .GetAwaiter().GetResult(); CachedGraph.Serializer.SerializeToFileAsync( GraphCacheFile.PipGraphId, PipGraph.SerializeGraphId, Path.Combine(OutputFilePath, nameof(GraphCacheFile.PipGraphId))) .GetAwaiter().GetResult(); Console.WriteLine($"Simulating [Reading]"); var simulator = new BuildSimulatorAnalyzer(Input); simulator.Increment = SimulatorIncrement ?? simulator.Increment; simulator.ExecutionData.DirectedGraph = m_mutableGraph; simulator.OutputDirectory = OutputFilePath; if (!simulator.ReadExecutionLog()) { Args.TruncatedXlgWarning(); } Console.WriteLine($"Simulating [Analyzing]"); simulator.Analyze(); Console.WriteLine($"Blocking Dependency Analysis"); DisplayTable <DepColumn> depTable = new DisplayTable <DepColumn>(" , "); foreach (var pipId in PipTable.Keys) { var pipType = PipTable.GetPipType(pipId); if (pipType == PipType.Process) { var entry = GetEntry(pipId); (PipId node, ulong cost)maxConsumedDependency = default; (PipId node, ulong cost)maxDependency = default; foreach (var dep in entry.PipDependencies) { var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.Key.ToNodeId()]; if (!maxDependency.node.IsValid || cost > maxDependency.cost) { maxDependency = (dep.Key, cost); } if (dep.Value != null && dep.Value.HasFlag(ContentFlag.Consumed)) { if (!maxConsumedDependency.node.IsValid || cost > maxConsumedDependency.cost) { maxConsumedDependency = (dep.Key, cost); } } } depTable.NextRow(); depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier}"); depTable.Set(DepColumn.MaxConsumedDependency, ToString(maxConsumedDependency.node)); depTable.Set(DepColumn.MaxConsumedDependencyChainCost, maxConsumedDependency.cost.ToMinutes()); depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node)); depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes()); } else if (pipType == PipType.SealDirectory && !PipTable.GetSealDirectoryKind(pipId).IsSourceSeal() && !IsSourceOnlySeal(pipId)) { var seal = (SealDirectory)GetPip(pipId); var entry = GetEntry(seal.Directory); (PipId node, ulong cost)maxDependency = default; foreach (var dep in DirectedGraph.GetIncomingEdges(pipId.ToNodeId())) { var cost = simulator.ExecutionData.BottomUpAggregateCosts[dep.OtherNode]; if (!maxDependency.node.IsValid || cost > maxDependency.cost) { maxDependency = (dep.OtherNode.ToPipId(), cost); } } depTable.NextRow(); depTable.Set(DepColumn.Id, $"{entry.SpecFileName}-{entry.Identifier} ({entry.FileCount} files)"); depTable.Set(DepColumn.MaxDependency, ToString(maxDependency.node)); depTable.Set(DepColumn.MaxDependencyChainCost, maxDependency.cost.ToMinutes()); depTable.Set(DepColumn.Directory, seal.DirectoryRoot.ToString(PathTable)); } } using (var blockAnalysisWriter = new StreamWriter(Path.Combine(OutputFilePath, "blockAnalysis.txt"))) { depTable.Write(blockAnalysisWriter); } m_writer.Dispose(); Console.WriteLine($"Analyzing complete"); return(0); }
/// <summary> /// Generate the output of failure data /// Failed pips + process chain : This points to the tool that failed. /// Set of dependencies diff contains: source and output changes. Which of those are directe referenced /// /// </summary> /// <param name="isDiff">There is diff data from another log</param> private void GenerateOutput(bool isDiff) { Console.WriteLine("Writing failed pip info to '{0}'.", OutputFilePath); using (var streamWriter = new StreamWriter(OutputFilePath)) using (JsonWriter writer = new JsonTextWriter(streamWriter)) { writer.WriteStartObject(); writer.WritePropertyName("Mounts"); { writer.WriteStartObject(); var mountPathExpander = m_mountPathExpander ?? CachedGraph.MountPathExpander; foreach (var mountRoot in mountPathExpander.GetAllRoots()) { var mount = mountPathExpander.GetSemanticPathInfo(mountRoot); writer.WritePropertyName(mount.RootName.ToString(StringTable)); writer.WriteValue(mountRoot.ToString(PathTable)); } writer.WriteEndObject(); } writer.WriteWhitespace(Environment.NewLine); writer.WritePropertyName("Failures"); { writer.WriteStartArray(); foreach (var failedPip in m_failedPips) { var pip = (Process)m_pipTable.HydratePip(failedPip, PipQueryContext.ViewerAnalyzer); writer.WriteWhitespace(Environment.NewLine); writer.WriteStartObject(); // prints the semistable hash. WritePropertyAndValue(writer, "PipId", ToDisplayString(failedPip)); WritePropertyAndValue(writer, "Working Directory", ToDisplayFilePath(pip.WorkingDirectory)); var provenance = pip.Provenance; WritePropertyAndValue( writer, "Qualifier", provenance != null ? PipGraph.Context.QualifierTable.GetCanonicalDisplayString(provenance.QualifierId) : string.Empty); WritePropertyAndValue( writer, "OutputValueSymbol", provenance != null ? provenance.OutputValueSymbol.ToString(SymbolTable) : string.Empty); var pipPerformance = m_pipPerformance[pip.PipId.Value]; WritePropertyAndValue(writer, "PeakMemoryUsageMb", pipPerformance.PeakMemoryUsageMb.ToString()); WritePropertyAndValue(writer, "NumberOfProcesses", pipPerformance.NumberOfProcesses.ToString()); WritePropertyAndValue( writer, "FileMonitoringViolationsNotWhitelisted", pipPerformance.FileMonitoringViolations.NumFileAccessViolationsNotWhitelisted.ToString()); if (isDiff) { WritePropertyAndValue(writer, "NumberOfImputChanges", GetDependencyChangesForPip(pip).ToString()); } if (m_pipDependencyViolationEventData.TryGetValue(pip.PipId, out var pipDependencyViolationEvent)) { foreach (var data in pipDependencyViolationEvent) { WritePropertyAndValue(writer, "DependencyViolationType", data.ViolationType.ToString()); WritePropertyAndValue(writer, "Path causing violation", ToDisplayFilePath(data.Path)); } } WriteProcessChain(writer, pip); // TODO : Are environment variables usefull for analysis // : Are are all the failures required or choose count as cmd line arg to take top n writer.WriteWhitespace(Environment.NewLine); writer.WriteEndObject(); } writer.WriteEndArray(); } writer.WriteWhitespace(Environment.NewLine); var fileToConsumerMap = m_fileToConsumerMap; var propertyName = "FileInfo"; if (isDiff) { propertyName = "FileInfoDiff"; fileToConsumerMap = m_fileToConsumerMapDiff; // If any added dependencies add them to the output if (m_fileToConsumerMapAdded.Count > 0) { WriteFileDependencies("FileInfoAdded", m_fileToConsumerMapAdded, writer); } // TODO:Add removed dependencies when compared to other log } WriteFileDependencies(propertyName, fileToConsumerMap, writer); writer.WriteWhitespace(Environment.NewLine); writer.WritePropertyName("PipGraph"); { writer.WriteStartObject(); m_failedPipsClosure.UnsafeReset(); writer.WritePropertyName("root"); { writer.WriteStartArray(); foreach (var failedPip in m_failedPips) { writer.WriteValue(ToDisplayString(failedPip)); } writer.WriteEndArray(); } List <NodeId> dependencyBuffer = new List <NodeId>(); nodeVisitor.VisitTransitiveDependencies( m_failedPips.Select(p => p.ToNodeId()), m_failedPipsClosure, visitNode: node => { dependencyBuffer.Clear(); foreach (var dependencyEdge in DataflowGraph.GetIncomingEdges(node)) { if (PipTable.GetPipType(dependencyEdge.OtherNode.ToPipId()) != PipType.HashSourceFile) { dependencyBuffer.Add(dependencyEdge.OtherNode); } } if (dependencyBuffer.Count != 0) { writer.WritePropertyName(ToDisplayString(node.ToPipId())); { writer.WriteStartArray(); foreach (var dependencyNode in dependencyBuffer) { writer.WriteValue(ToDisplayString(dependencyNode.ToPipId())); } writer.WriteEndArray(); } } return(true); }); writer.WriteEndObject(); } writer.WriteEndObject(); } }
public PipQueueTestExecutionEnvironment(BuildXLContext context, IConfiguration configuration, PipTable pipTable, IKextConnection sandboxedKextConnection = null) { Contract.Requires(context != null); Contract.Requires(configuration != null); Context = context; LoggingContext = CreateLoggingContextForTest(); Configuration = configuration; FileContentTable = FileContentTable.CreateNew(); ContentFingerprinter = new PipContentFingerprinter( context.PathTable, artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo, ExtraFingerprintSalts.Default(), pathExpander: PathExpander); PipTable = pipTable; PipFragmentRenderer = this.CreatePipFragmentRenderer(); IpcProvider = IpcFactory.GetProvider(); var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext); Cache = InMemoryCacheFactory.Create(); LocalDiskContentStore = new LocalDiskContentStore(LoggingContext, context.PathTable, FileContentTable, tracker); m_sandboxedKextConnection = sandboxedKextConnection; m_expectedWrittenContent = new ConcurrentDictionary <FileArtifact, ContentHash>(); m_wellKnownFiles = new ConcurrentDictionary <FileArtifact, ContentHash>(); m_producers = new ConcurrentDictionary <FileArtifact, Pip>(); m_filesystemView = new TestPipGraphFilesystemView(Context.PathTable); var fileSystemView = new FileSystemView(Context.PathTable, m_filesystemView, LocalDiskContentStore); State = new PipExecutionState( configuration, cache: new PipTwoPhaseCache(LoggingContext, Cache, context, PathExpander), unsafeConfiguration: configuration.Sandbox.UnsafeSandboxConfiguration, preserveOutputsSalt: ContentHashingUtilities.CreateRandom(), fileAccessWhitelist: FileAccessWhitelist, directoryMembershipFingerprinter: this, pathExpander: PathExpander, executionLog: null, fileSystemView: fileSystemView, fileContentManager: new FileContentManager(this, new NullOperationTracker()), directoryMembershipFinterprinterRuleSet: null); m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>(); ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable); }
public TestEnv( string name, string rootPath, List <IMount> mounts = null, PathTable pathTable = null, Action <CommandLineConfiguration> customizeConfig = null) { Contract.Requires(name != null); Contract.Requires(!string.IsNullOrEmpty(rootPath)); LoggingContext = new LoggingContext("TestLogger." + name); PathTable = pathTable ?? new PathTable(); PipDataBuilderPool = new ObjectPool <PipDataBuilder>(() => new PipDataBuilder(PathTable.StringTable), _ => { }); // The tests that use TestEnv need to be modernized to take a filesystem var fileSystem = new PassThroughFileSystem(PathTable); Context = EngineContext.CreateNew(CancellationToken.None, PathTable, fileSystem); // Add some well-known paths with fixed casing to the Context.PathTable AbsolutePath.Create(Context.PathTable, rootPath.ToLowerInvariant()); var root = AbsolutePath.Create(Context.PathTable, rootPath); var configuration = ConfigHelpers.CreateDefaultForXml(Context.PathTable, root); configuration.Layout.SourceDirectory = root.Combine(PathTable, PathAtom.Create(PathTable.StringTable, "src")); // These tests have non-standard src folder configuration.Engine.MaxRelativeOutputDirectoryLength = 260; configuration.Schedule.EnableLazyOutputMaterialization = false; configuration.Schedule.UnsafeDisableGraphPostValidation = false; configuration.Schedule.ComputePipStaticFingerprints = true; configuration.Sandbox.FileAccessIgnoreCodeCoverage = true; customizeConfig?.Invoke(configuration); BuildXLEngine.PopulateFileSystemCapabilities(configuration, configuration, Context.PathTable, LoggingContext); BuildXLEngine.PopulateLoggingAndLayoutConfiguration(configuration, Context.PathTable, bxlExeLocation: null, inTestMode: true); BuildXLEngine.PopulateAndValidateConfiguration(configuration, configuration, Context.PathTable, LoggingContext); Configuration = configuration; var mountsTable = MountsTable.CreateAndRegister(LoggingContext, Context, Configuration, null); if (mounts != null) { foreach (var mount in mounts) { mountsTable.AddResolvedMount(mount); } } AbsolutePath specFile = SourceRoot.CreateRelative(Context.PathTable, "TestSpecFile.dsc"); var graph = TestSchedulerFactory.CreateEmptyPipGraph(Context, configuration, mountsTable.MountPathExpander); PipTable = graph.PipTable; PipGraph = graph; var locationData = new LocationData(specFile, 0, 0); var modulePip = ModulePip.CreateForTesting(Context.StringTable, specFile); PipGraph.AddModule(modulePip); PipGraph.AddSpecFile(new SpecFilePip(FileArtifact.CreateSourceFile(specFile), locationData, modulePip.Module)); PipConstructionHelper = PipConstructionHelper.CreateForTesting( Context, ObjectRoot, redirectedRoot: Configuration.Layout.RedirectedDirectory, pipGraph: PipGraph, moduleName: modulePip.Identity.ToString(Context.StringTable), symbol: name, specPath: specFile); Paths = new Paths(PathTable); mountsTable.CompleteInitialization(); }
/// <summary> /// Creates a <see cref="FingerprintStoreExecutionLogTarget"/>. /// </summary> /// <returns> /// If successful, a <see cref="FingerprintStoreExecutionLogTarget"/> that logs to /// a <see cref="Tracing.FingerprintStore"/> at the provided directory; /// otherwise, null. /// </returns> public static FingerprintStoreExecutionLogTarget Create( PipExecutionContext context, PipTable pipTable, PipContentFingerprinter pipContentFingerprinter, LoggingContext loggingContext, IConfiguration configuration, EngineCache cache, IReadonlyDirectedGraph graph, CounterCollection <FingerprintStoreCounters> counters, IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance = null, FingerprintStoreTestHooks testHooks = null) { var fingerprintStorePathString = configuration.Layout.FingerprintStoreDirectory.ToString(context.PathTable); var cacheLookupFingerprintStorePathString = configuration.Logging.CacheLookupFingerprintStoreLogDirectory.ToString(context.PathTable); try { FileUtilities.CreateDirectoryWithRetry(fingerprintStorePathString); } catch (BuildXLException ex) { Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message); throw new BuildXLException("Unable to create fingerprint store directory: ", ex); } var maxEntryAge = new TimeSpan(hours: 0, minutes: configuration.Logging.FingerprintStoreMaxEntryAgeMinutes, seconds: 0); var possibleExecutionStore = FingerprintStore.Open( fingerprintStorePathString, maxEntryAge: maxEntryAge, mode: configuration.Logging.FingerprintStoreMode, loggingContext: loggingContext, counters: counters, testHooks: testHooks); Possible <FingerprintStore> possibleCacheLookupStore = new Failure <string>("No attempt to create a cache lookup fingerprint store yet."); if (configuration.Logging.FingerprintStoreMode != FingerprintStoreMode.ExecutionFingerprintsOnly) { try { FileUtilities.CreateDirectoryWithRetry(cacheLookupFingerprintStorePathString); } catch (BuildXLException ex) { Logger.Log.FingerprintStoreUnableToCreateDirectory(loggingContext, fingerprintStorePathString, ex.Message); throw new BuildXLException("Unable to create fingerprint store directory: ", ex); } possibleCacheLookupStore = FingerprintStore.Open( cacheLookupFingerprintStorePathString, maxEntryAge: maxEntryAge, mode: configuration.Logging.FingerprintStoreMode, loggingContext: loggingContext, counters: counters, testHooks: testHooks); } if (possibleExecutionStore.Succeeded && (possibleCacheLookupStore.Succeeded || configuration.Logging.FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly)) { return(new FingerprintStoreExecutionLogTarget( loggingContext, context, pipTable, pipContentFingerprinter, possibleExecutionStore.Result, possibleCacheLookupStore.Succeeded ? possibleCacheLookupStore.Result : null, configuration, cache, graph, counters, runnablePipPerformance)); } else { if (!possibleExecutionStore.Succeeded) { Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleExecutionStore.Failure.DescribeIncludingInnerFailures()); } if (!possibleCacheLookupStore.Succeeded) { Logger.Log.FingerprintStoreUnableToOpen(loggingContext, possibleCacheLookupStore.Failure.DescribeIncludingInnerFailures()); } } return(null); }
private IEnumerable <PipReference> AllPipIds(PipTable pipTable) { return(pipTable.StableKeys .Where(pipId => pipTable.GetPipType(pipId) != PipType.HashSourceFile) .Select(pipId => new PipReference(pipTable, pipId, PipQueryContext.Explorer))); }
public override int Analyze() { Console.WriteLine("Writing failed pip info to '{0}'.", OutputFilePath); using (var streamWriter = new StreamWriter(OutputFilePath)) using (JsonWriter writer = new JsonTextWriter(streamWriter)) { writer.WriteStartObject(); writer.WritePropertyName("Mounts"); { writer.WriteStartObject(); var mountPathExpander = m_mountPathExpander ?? CachedGraph.MountPathExpander; foreach (var mountRoot in mountPathExpander.GetAllRoots()) { var mount = mountPathExpander.GetSemanticPathInfo(mountRoot); writer.WritePropertyName(mount.RootName.ToString(StringTable)); writer.WriteValue(mountRoot.ToString(PathTable)); } writer.WriteEndObject(); } writer.WritePropertyName("FileInfo"); { writer.WriteStartObject(); foreach (var fileEntry in m_fileToConsumerMap) { var file = fileEntry.Key; var consumers = fileEntry.Value; var path = ToDisplayFilePath(file); if (path != null) { writer.WritePropertyName(path, true); { writer.WriteStartArray(); foreach (var consumer in consumers) { writer.WriteValue(ToDisplayString(consumer)); } writer.WriteEndArray(); } } } writer.WriteEndObject(); } writer.WritePropertyName("PipGraph"); { writer.WriteStartObject(); m_failedPipsClosure.UnsafeReset(); writer.WritePropertyName("root"); { writer.WriteStartArray(); foreach (var failedPip in m_failedPips) { writer.WriteValue(ToDisplayString(failedPip)); } writer.WriteEndArray(); } List <NodeId> dependencyBuffer = new List <NodeId>(); nodeVisitor.VisitTransitiveDependencies(m_failedPips.Select(p => p.ToNodeId()), m_failedPipsClosure, visitNode: node => { dependencyBuffer.Clear(); foreach (var dependencyEdge in DataflowGraph.GetIncomingEdges(node)) { if (PipTable.GetPipType(dependencyEdge.OtherNode.ToPipId()) != PipType.HashSourceFile) { dependencyBuffer.Add(dependencyEdge.OtherNode); } } if (dependencyBuffer.Count != 0) { writer.WritePropertyName(ToDisplayString(node.ToPipId())); { writer.WriteStartArray(); foreach (var dependencyNode in dependencyBuffer) { writer.WriteValue(ToDisplayString(dependencyNode.ToPipId())); } writer.WriteEndArray(); } } return(true); }); writer.WriteEndObject(); } writer.WriteEndObject(); } return(0); }
/// <nodoc /> public static Xldb.Proto.PipGraph ToPipGraph(this PipGraph pipGraph, PathTable pathTable, PipTable pipTable, NameExpander nameExpander) { var xldbPipGraph = new Xldb.Proto.PipGraph() { GraphId = pipGraph.GraphId.ToString(), SemistableFingerprint = new ContentFingerprint() { Hash = pipGraph.SemistableFingerprint.Hash.ToFingerprint() }, NodeCount = pipTable.StableKeys.Count, MaxAbsolutePathIndex = pipGraph.MaxAbsolutePathIndex, FileCount = pipGraph.FileCount, ContentCount = pipGraph.ContentCount, ArtifactContentCount = pipGraph.ArtifactContentCount, ApiServerMoniker = pipGraph.ApiServerMoniker.ToString(pathTable), }; xldbPipGraph.AllSealDirectoriesAndProducers.AddRange(pipGraph.AllSealDirectoriesAndProducers.Select(kvp => new DirectoryArtifactMap() { Artifact = kvp.Key.ToDirectoryArtifact(pathTable, nameExpander), PipId = kvp.Value.Value })); xldbPipGraph.OutputDirectoryExclusions.AddRange( pipGraph.OutputDirectoryExclusions.UnsafeGetList().Select(p => p.ToAbsolutePath(pathTable, nameExpander))); xldbPipGraph.StableKeys.AddRange(pipTable.StableKeys.Select(stableKey => stableKey.Value)); foreach (var kvp in pipGraph.Modules) { xldbPipGraph.Modules.Add(kvp.Key.Value.ToString(pathTable), kvp.Value.Value); } return(xldbPipGraph); }
/// <summary> /// Analyzes cache misses. /// </summary> public override int Analyze() { Console.WriteLine("Starting fingerprint store analysis at: " + DateTime.Now.ToString()); if (!m_newReader.TryGetCacheMissList(out var cacheMissList)) { WriteLine("Could not retrieve cache miss list for analysis. There may have been a failure during the build."); return(0); } Console.WriteLine("Finished getting list of pips that had a cache miss at: " + DateTime.Now.ToString()); if (m_oldReader.StoreVersion != m_newReader.StoreVersion) { WriteLine($"WARNING: Format version numbers of the fingerprint store do not match. Old: {m_oldReader.StoreVersion}, New: {m_newReader.StoreVersion}."); } if (SemiStableHashToRun != -1) { var firstMiss = cacheMissList.FirstOrDefault(x => PipTable.GetPipSemiStableHash(x.PipId) == SemiStableHashToRun); if (firstMiss.CacheMissType == PipCacheMissType.Invalid) { foreach (var pipId in PipTable.StableKeys) { var possibleMatch = PipTable.GetPipSemiStableHash(pipId); if (possibleMatch == SemiStableHashToRun) { firstMiss = new PipCacheMissInfo(pipId, PipCacheMissType.Hit); } } } Console.WriteLine("Analyzing single pip starting at: " + DateTime.Now.ToString()); AnalyzePip(firstMiss, HydratePip(firstMiss.PipId), m_writer); } else { var frontierCacheMissList = new List <PipCacheMissInfo>(); foreach (var miss in cacheMissList) { if (m_model.HasChangedDependencies(miss.PipId) && !AllPips) { continue; } frontierCacheMissList.Add(miss); m_model.MarkChanged(miss.PipId); } Console.WriteLine("Finding frontier pips done at " + DateTime.Now.ToString()); int i = 0; foreach (var miss in frontierCacheMissList) { if (i % 10 == 0) { Console.WriteLine("Done " + i + " of " + cacheMissList.Count); } var pip = HydratePip(miss.PipId); WriteLine($"================== Analyzing pip ========================"); AnalyzePip(miss, pip, m_writer); WriteLine("================== Complete pip ========================"); WriteLine(); i++; } } return(0); }
private string ToDisplayString(PipId pipId) { return("pip" + PipTable.GetPipSemiStableHash(pipId).ToString("X16", CultureInfo.InvariantCulture)); }
/// <summary> /// Creates an execution environment for a single pip. To run pips incrementally, the <paramref name="fileContentTable"/> and <paramref name="pipCache"/> should be specified. /// </summary> public DummyPipExecutionEnvironment( LoggingContext loggingContext, PipExecutionContext context, IConfiguration config, FileContentTable fileContentTable = null, EngineCache pipCache = null, SemanticPathExpander semanticPathExpander = null, PipContentFingerprinter.PipDataLookup pipDataLookup = null, FileAccessWhitelist fileAccessWhitelist = null, bool allowUnspecifiedSealedDirectories = false, PipTable pipTable = null, IIpcProvider ipcProvider = null, IKextConnection sandboxedKextConnection = null) { Contract.Requires(context != null); Contract.Requires(config != null); LoggingContext = loggingContext; Context = context; // Ensure paths visible when debugging PathTable.DebugPathTable = Context.PathTable; Configuration = config; PipTable = pipTable; PathExpander = semanticPathExpander ?? SemanticPathExpander.Default; ContentFingerprinter = new PipContentFingerprinter( Context.PathTable, artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo, new ExtraFingerprintSalts(config, PipFingerprintingVersion.TwoPhaseV2, fingerprintSalt: null, searchPathToolsHash: null), pathExpander: PathExpander, pipDataLookup: pipDataLookup); PipFragmentRenderer = this.CreatePipFragmentRenderer(); IpcProvider = ipcProvider ?? IpcFactory.GetProvider(); FileContentTable = fileContentTable ?? FileContentTable.CreateNew(); Cache = pipCache; FileAccessWhitelist = fileAccessWhitelist; m_allowUnspecifiedSealedDirectories = allowUnspecifiedSealedDirectories; m_sandboxedKextConnection = sandboxedKextConnection; if (Cache == null) { Cache = InMemoryCacheFactory.Create(); } var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext); LocalDiskContentStore = new LocalDiskContentStore(loggingContext, context.PathTable, FileContentTable, tracker); PipGraphView = new TestPipGraphFilesystemView(Context.PathTable); m_operationTracker = new OperationTracker(loggingContext); var fileSystemView = new FileSystemView(Context.PathTable, PipGraphView, LocalDiskContentStore); var preserveOutputsSalt = UnsafeOptions.PreserveOutputsNotUsed; if (config.Sandbox.UnsafeSandboxConfiguration.PreserveOutputs != PreserveOutputsMode.Disabled) { preserveOutputsSalt = ContentHashingUtilities.HashString(Guid.NewGuid().ToString()); } State = new PipExecutionState( config, cache: new PipTwoPhaseCache(loggingContext, Cache, context, PathExpander), fileAccessWhitelist: FileAccessWhitelist, directoryMembershipFingerprinter: this, pathExpander: PathExpander, executionLog: ExecutionLogRecorder, fileSystemView: fileSystemView, fileContentManager: GetFileContentManager(), directoryMembershipFinterprinterRuleSet: null, unsafeConfiguration: config.Sandbox.UnsafeSandboxConfiguration, preserveOutputsSalt: preserveOutputsSalt, serviceManager: new DummyServiceManager()); m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>(); ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable); }
public RequiredDependencyAnalyzer(AnalysisInput input) : base(input) { m_pool = new ObjectPool <WorkerAnalyzer>(() => CreateAnalyzer(), w => { }); m_pipTable = EngineSchedule.CreateEmptyPipTable(input.CachedGraph.Context); }
public async Task Stress() { const int N = 5; const int M = N * N; var context = BuildXLContext.CreateInstanceForTesting(); var loggingContext = CreateLoggingContextForTest(); var pathTable = context.PathTable; using (var tempFiles = new TempFileStorage(canGetFileNames: true)) { var config = ConfigHelpers.CreateDefault(pathTable, tempFiles.GetUniqueFileName(), tempFiles); using (var pipTable = new PipTable( context.PathTable, context.SymbolTable, initialBufferSize: 1024, maxDegreeOfParallelism: (Environment.ProcessorCount + 2) / 3, debug: false)) { var executionEnvironment = new PipQueueTestExecutionEnvironment(context, config, pipTable, GetSandboxedKextConnection()); Func <RunnablePip, Task <PipResult> > taskFactory = async(runnablePip) => { PipResult result; var operationTracker = new OperationTracker(runnablePip.LoggingContext); var pip = runnablePip.Pip; using (var operationContext = operationTracker.StartOperation(PipExecutorCounter.PipRunningStateDuration, pip.PipId, pip.PipType, runnablePip.LoggingContext)) { result = await TestPipExecutor.ExecuteAsync(operationContext, executionEnvironment, pip); } executionEnvironment.MarkExecuted(pip); return(result); }; string executable = CmdHelper.OsShellExe; FileArtifact executableArtifact = FileArtifact.CreateSourceFile(AbsolutePath.Create(pathTable, executable)); // This is the only file artifact we reference without a producer. Rather than scheduling a hashing pip, let's just invent one (so fingerprinting can succeed). executionEnvironment.AddWellKnownFile(executableArtifact, WellKnownContentHashes.UntrackedFile); using (var phase1PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule)) { // phase 1: create some files var baseFileArtifacts = new List <FileArtifact>(); for (int i = 0; i < N; i++) { string destination = tempFiles.GetUniqueFileName(); AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination); FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion(); baseFileArtifacts.Add(destinationArtifact); PipData contents = PipDataBuilder.CreatePipData( context.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, i.ToString(CultureInfo.InvariantCulture)); var writeFile = new WriteFile(destinationArtifact, contents, WriteFileEncoding.Utf8, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context)); var pipId = pipTable.Add((uint)(i + 1), writeFile); var contentHash = ContentHashingUtilities.HashString(contents.ToString(pathTable)); executionEnvironment.AddExpectedWrite(writeFile, destinationArtifact, contentHash); var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactory, 0); runnable.Start(new OperationTracker(loggingContext), loggingContext); runnable.SetDispatcherKind(DispatcherKind.IO); phase1PipQueue.Enqueue(runnable); } phase1PipQueue.SetAsFinalized(); phase1PipQueue.DrainQueues(); await Task.WhenAll( Enumerable.Range(0, 2).Select( async range => { using (var phase2PipQueue = new PipQueue(executionEnvironment.Configuration.Schedule)) { // phase 2: do some more with those files var pips = new ConcurrentDictionary <PipId, Tuple <string, int> >(); var checkerTasks = new ConcurrentQueue <Task>(); Action <PipId, Task <PipResult> > callback = (id, task) => { XAssert.IsTrue(task.Status == TaskStatus.RanToCompletion); XAssert.IsFalse(task.Result.Status.IndicatesFailure()); Tuple <string, int> t; if (!pips.TryRemove(id, out t)) { XAssert.Fail(); } checkerTasks.Enqueue( Task.Run( () => { string actual = File.ReadAllText(t.Item1).Trim(); // TODO: Make this async XAssert.AreEqual(actual, t.Item2.ToString()); })); }; var r = new Random(0); for (int i = 0; i < M; i++) { int sourceIndex = r.Next(baseFileArtifacts.Count); FileArtifact sourceArtifact = baseFileArtifacts[sourceIndex]; string destination = tempFiles.GetUniqueFileName(); AbsolutePath destinationAbsolutePath = AbsolutePath.Create(pathTable, destination); FileArtifact destinationArtifact = FileArtifact.CreateSourceFile(destinationAbsolutePath).CreateNextWrittenVersion(); Pip pip; DispatcherKind queueKind; switch (r.Next(2)) { case 0: pip = new CopyFile(sourceArtifact, destinationArtifact, ReadOnlyArray <StringId> .Empty, PipProvenance.CreateDummy(context)); queueKind = DispatcherKind.IO; executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact)); break; case 1: string workingDirectory = OperatingSystemHelper.IsUnixOS ? "/tmp" : Environment.GetFolderPath(Environment.SpecialFolder.Windows); AbsolutePath workingDirectoryAbsolutePath = AbsolutePath.Create(pathTable, workingDirectory); var pipData = OperatingSystemHelper.IsUnixOS ? PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "-c", "'", "cp", sourceArtifact, destinationArtifact, "'") : PipDataBuilder.CreatePipData(pathTable.StringTable, " ", PipDataFragmentEscaping.CRuntimeArgumentRules, "/d", "/c", "copy", "/B", sourceArtifact, destinationArtifact); queueKind = DispatcherKind.CPU; pip = new Process( executableArtifact, workingDirectoryAbsolutePath, pipData, FileArtifact.Invalid, PipData.Invalid, ReadOnlyArray <EnvironmentVariable> .Empty, FileArtifact.Invalid, FileArtifact.Invalid, FileArtifact.Invalid, tempFiles.GetUniqueDirectory(pathTable), null, null, ReadOnlyArray <FileArtifact> .FromWithoutCopy(executableArtifact, sourceArtifact), ReadOnlyArray <FileArtifactWithAttributes> .FromWithoutCopy(destinationArtifact.WithAttributes()), ReadOnlyArray <DirectoryArtifact> .Empty, ReadOnlyArray <DirectoryArtifact> .Empty, ReadOnlyArray <PipId> .Empty, ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencies(pathTable)), ReadOnlyArray <AbsolutePath> .From(CmdHelper.GetCmdDependencyScopes(pathTable)), ReadOnlyArray <StringId> .Empty, ReadOnlyArray <int> .Empty, ReadOnlyArray <ProcessSemaphoreInfo> .Empty, provenance: PipProvenance.CreateDummy(context), toolDescription: StringId.Invalid, additionalTempDirectories: ReadOnlyArray <AbsolutePath> .Empty); executionEnvironment.AddExpectedWrite(pip, destinationArtifact, executionEnvironment.GetExpectedContent(sourceArtifact)); break; default: Contract.Assert(false); continue; } var pipId = pipTable.Add((uint)((range *M) + N + i + 1), pip); Func <RunnablePip, Task> taskFactoryWithCallback = async(runnablePip) => { var task = taskFactory(runnablePip); var pipResult = await task; callback(pipId, task); }; var runnable = RunnablePip.Create(loggingContext, executionEnvironment, pipId, pipTable.GetPipType(pipId), 0, taskFactoryWithCallback, 0); runnable.Start(new OperationTracker(loggingContext), loggingContext); runnable.SetDispatcherKind(queueKind); phase2PipQueue.Enqueue(runnable); if (!pips.TryAdd(pipId, Tuple.Create(destination, sourceIndex))) { Contract.Assert(false); } } phase2PipQueue.SetAsFinalized(); phase2PipQueue.DrainQueues(); XAssert.AreEqual(0, pips.Count); await Task.WhenAll(checkerTasks); } })); } } } }
public PipsAnalyzer(AnalysisInput input) : base(input) { m_pipTable = input.CachedGraph.PipTable; }