Пример #1
0
        /// <summary>
        /// Gets all scheduled pips (ie. all the non meta pips). Must return pair of PipType, IMessage so user knows how to
        /// cast the IMessage object
        /// </summary>
        public IEnumerable <(PipType, IMessage)> GetAllScheduledPips()
        {
            Contract.Requires(Accessor != null, "XldbDataStore is not initialized");

            var allPips = new List <(PipType, IMessage)>();

            // Empty key to prefix match all pips stored in DB
            var pipIdKey = new PipIdKey();

            var maybeFound = Accessor.Use(database =>
            {
                foreach (var kvp in database.PrefixSearch(pipIdKey.ToByteArray(), PipColumnFamilyName))
                {
                    var pipType = PipIdKey.Parser.ParseFrom(kvp.Key).PipType;
                    if (m_pipParserDictionary.TryGetValue(pipType, out var parser))
                    {
                        var xldbPip = parser.ParseFrom(kvp.Value);
                        allPips.Add((pipType, xldbPip));
                    }
                    else
                    {
                        Contract.Assert(false, "No parser found for PipId");
                    }
                }
            });

            if (!maybeFound.Succeeded)
            {
                maybeFound.Failure.Throw();
            }

            return(allPips);
        }
Пример #2
0
        /// <summary>
        /// Gets all pips of a certain type.
        /// </summary>
        /// <returns>Returns list of all pips of certain type, empty if no such pips exist.</returns>
        public IEnumerable <IMessage> GetAllPipsByType(PipType pipType)
        {
            Contract.Requires(Accessor != null, "XldbDataStore is not initialized");

            var storedPips = new List <IMessage>();

            if (!m_pipParserDictionary.TryGetValue(pipType, out var parser))
            {
                Contract.Assert(false, "No parser found for PipId");
            }

            // Empty key will match all pips in the prefix search, and then we grab only the ones that match the type we want
            var pipIdKey = new PipIdKey();

            var maybeFound = Accessor.Use(database =>
            {
                foreach (var kvp in database.PrefixSearch(pipIdKey.ToByteArray(), PipColumnFamilyName))
                {
                    var pipKey = PipIdKey.Parser.ParseFrom(kvp.Key);
                    if (pipKey.PipType == pipType)
                    {
                        storedPips.Add(parser.ParseFrom(kvp.Value));
                    }
                }
            });

            if (!maybeFound.Succeeded)
            {
                maybeFound.Failure.Throw();
            }

            return(storedPips);
        }
Пример #3
0
        /// <summary>
        /// Gets the pip stored based on the pip id
        /// </summary>
        /// <returns>Returns null if no such pip is found</returns>
        public IMessage GetPipByPipId(uint pipId, out PipType pipType)
        {
            Contract.Requires(Accessor != null, "XldbDataStore is not initialized");

            IMessage foundPip = null;

            var pipIdKey = new PipIdKey()
            {
                PipId = pipId
            };

            PipType outPipType = 0;

            var maybeFound = Accessor.Use(database =>
            {
                foreach (var kvp in database.PrefixSearch(pipIdKey.ToByteArray(), PipColumnFamilyName))
                {
                    var pipKey = PipIdKey.Parser.ParseFrom(kvp.Key);
                    if (m_pipParserDictionary.TryGetValue(pipKey.PipType, out var parser))
                    {
                        foundPip   = parser.ParseFrom(kvp.Value);
                        outPipType = pipKey.PipType;
                    }
                    else
                    {
                        Contract.Assert(false, "No parser found for PipId");
                    }
                }
            });

            if (!maybeFound.Succeeded)
            {
                maybeFound.Failure.Throw();
            }

            pipType = outPipType;

            return(foundPip);
        }
Пример #4
0
        /// <summary>
        /// Ingest all of the pips to RocksDB
        /// </summary>
        private void IngestAllPips()
        {
            var totalNumberOfPips = CachedGraph.PipTable.StableKeys.Count;
            var pipIds            = CachedGraph.PipTable.StableKeys;
            var concurrency       = (int)(Environment.ProcessorCount * s_innerConcurrencyMultiplier);
            var partitionSize     = totalNumberOfPips / concurrency;

            Console.WriteLine("Ingesting pips now ...");

            Parallel.For(0, concurrency, i =>
            {
                // Hold only one lock while inserting all of these keys into the DB
                var maybeInserted = m_accessor.Use(database =>
                {
                    var start            = i * partitionSize;
                    var end              = (i + 1) == concurrency ? totalNumberOfPips : (i + 1) * partitionSize;
                    var pipsIngested     = 0;
                    var pipSemistableMap = new Dictionary <byte[], byte[]>();
                    var pipIdMap         = new Dictionary <byte[], byte[]>();

                    for (int j = start; j < end; j++)
                    {
                        var pipId = pipIds[j];
                        pipsIngested++;

                        if (pipsIngested % s_pipOutputBatchLogSize == 0)
                        {
                            Console.Write(".");
                            database.ApplyBatch(pipSemistableMap);
                            database.ApplyBatch(pipIdMap, XldbDataStore.PipColumnFamilyName);

                            pipSemistableMap.Clear();
                            pipIdMap.Clear();
                        }

                        var hydratedPip = CachedGraph.PipTable.HydratePip(pipId, BuildXL.Pips.PipQueryContext.PipGraphRetrieveAllPips);
                        var pipType     = hydratedPip.PipType;

                        if (pipType == PipType.Value || pipType == PipType.HashSourceFile || pipType == PipType.SpecFile || pipType == PipType.Module)
                        {
                            continue;
                        }

                        var xldbPip  = hydratedPip.ToPip(CachedGraph);
                        var pipIdKey = new PipIdKey()
                        {
                            PipId   = hydratedPip.PipId.Value,
                            PipType = (Xldb.Proto.PipType)(pipType + 1)
                        };
                        var pipIdKeyArr          = pipIdKey.ToByteArray();
                        IMessage xldbSpecificPip = xldbPip;

                        if (pipType == PipType.Ipc)
                        {
                            var ipcPip      = (Pips.Operations.IpcPip)hydratedPip;
                            xldbSpecificPip = ipcPip.ToIpcPip(PathTable, xldbPip, m_nameExpander);

                            foreach (var fileArtifact in ipcPip.FileDependencies)
                            {
                                AddToFileConsumerMap(fileArtifact, pipId);
                            }

                            foreach (var directoryArtifact in ipcPip.DirectoryDependencies)
                            {
                                AddToDirectoryConsumerMap(directoryArtifact, pipId);
                            }

                            AddToDbStorageDictionary(DBStoredTypes.IpcPip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
                        }
                        else if (pipType == PipType.SealDirectory)
                        {
                            var sealDirectoryPip = (Pips.Operations.SealDirectory)hydratedPip;
                            xldbSpecificPip      = sealDirectoryPip.ToSealDirectory(PathTable, xldbPip, m_nameExpander);

                            // If it is a shared opaque, then flatten the list of composted directories
                            if (sealDirectoryPip.Kind == Pips.Operations.SealDirectoryKind.SharedOpaque)
                            {
                                var directoryQueue = new Queue <Utilities.DirectoryArtifact>();

                                foreach (var initialDirectory in sealDirectoryPip.ComposedDirectories)
                                {
                                    directoryQueue.Enqueue(initialDirectory);
                                }

                                while (directoryQueue.Count > 0)
                                {
                                    var nestedDirectory = directoryQueue.Dequeue();
                                    var nestedPipId     = CachedGraph.PipGraph.GetSealedDirectoryNode(nestedDirectory).ToPipId();

                                    if (CachedGraph.PipTable.IsSealDirectoryComposite(nestedPipId))
                                    {
                                        var nestedPip = (Pips.Operations.SealDirectory)CachedGraph.PipGraph.GetSealedDirectoryPip(nestedDirectory, BuildXL.Pips.PipQueryContext.SchedulerExecuteSealDirectoryPip);
                                        foreach (var pendingDirectory in nestedPip.ComposedDirectories)
                                        {
                                            directoryQueue.Enqueue(pendingDirectory);
                                        }
                                    }
                                    else
                                    {
                                        Contract.Assert(nestedDirectory.IsOutputDirectory());
                                        AddToDirectoryConsumerMap(nestedDirectory, pipId);
                                    }
                                }
                            }

                            foreach (var fileArtifact in sealDirectoryPip.Contents)
                            {
                                AddToFileConsumerMap(fileArtifact, pipId);
                            }

                            AddToDbStorageDictionary(DBStoredTypes.SealDirectoryPip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
                        }
                        else if (pipType == PipType.CopyFile)
                        {
                            var copyFilePip = (Pips.Operations.CopyFile)hydratedPip;
                            xldbSpecificPip = copyFilePip.ToCopyFile(PathTable, xldbPip, m_nameExpander);
                            AddToFileConsumerMap(copyFilePip.Source, pipId);
                            AddToDbStorageDictionary(DBStoredTypes.CopyFilePip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
                        }
                        else if (pipType == PipType.WriteFile)
                        {
                            var writeFilePip = (Pips.Operations.WriteFile)hydratedPip;
                            xldbSpecificPip  = writeFilePip.ToWriteFile(PathTable, xldbPip, m_nameExpander);
                            AddToDbStorageDictionary(DBStoredTypes.WriteFilePip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
                        }
                        else if (pipType == PipType.Process)
                        {
                            var processPip  = (Pips.Operations.Process)hydratedPip;
                            xldbSpecificPip = processPip.ToProcessPip(PathTable, xldbPip, m_nameExpander);

                            AddToFileConsumerMap(processPip.StandardInputFile, pipId);
                            AddToFileConsumerMap(processPip.Executable, pipId);

                            foreach (var fileArtifact in processPip.Dependencies)
                            {
                                AddToFileConsumerMap(fileArtifact, pipId);
                            }

                            foreach (var directoryArtifact in processPip.DirectoryDependencies)
                            {
                                AddToDirectoryConsumerMap(directoryArtifact, pipId);
                            }

                            AddToDbStorageDictionary(DBStoredTypes.ProcessPip, pipIdKeyArr.Length + xldbSpecificPip.ToByteArray().Length);
                        }
                        else
                        {
                            Contract.Assert(false, "Unknown pip type parsed");
                        }

                        // If the SemiStableHash != 0, then we want to create the SemistableHash -> PipId indirection.
                        // Else we do not want that since the key would no longer be unique
                        if (hydratedPip.SemiStableHash != 0)
                        {
                            var pipSemistableHashKey = new PipSemistableHashKey()
                            {
                                SemiStableHash = hydratedPip.SemiStableHash,
                            };

                            pipSemistableMap.Add(pipSemistableHashKey.ToByteArray(), pipIdKeyArr);
                        }

                        pipIdMap.Add(pipIdKeyArr, xldbSpecificPip.ToByteArray());
                    }

                    database.ApplyBatch(pipSemistableMap);
                    database.ApplyBatch(pipIdMap, XldbDataStore.PipColumnFamilyName);
                });

                if (!maybeInserted.Succeeded)
                {
                    Console.WriteLine("Failed to insert pip data into RocksDb. Exiting analyzer now ...");
                    maybeInserted.Failure.Throw();
                }
            });
        }