/// <summary> /// Gets producers or consumers of a file based on the key passed in /// </summary> private IEnumerable <uint> GetProducerConsumerOfFileByKey(FileProducerConsumerKey key) { Contract.Requires(Accessor != null, "XldbDataStore is not initialized"); var fileProducersOrConsumers = new List <uint>(); var maybeFound = Accessor.Use(database => { foreach (var kvp in database.PrefixSearch(key.ToByteArray(), StaticGraphColumnFamilyName)) { if (key.Type == ProducerConsumerType.Producer) { var pipId = FileProducerValue.Parser.ParseFrom(kvp.Value).PipId; fileProducersOrConsumers.Add(pipId); } else if (key.Type == ProducerConsumerType.Consumer) { fileProducersOrConsumers.AddRange(FileConsumerValue.Parser.ParseFrom(kvp.Value).PipIds); } } }); if (!maybeFound.Succeeded) { maybeFound.Failure.Throw(); } return(fileProducersOrConsumers); }
/// <summary> /// Gets all consumers of a particular file /// </summary> public IEnumerable <uint> GetConsumersOfFile(string path) { Contract.Requires(Accessor != null, "XldbDataStore is not initialized"); var fileConsumerKey = new FileProducerConsumerKey() { Type = ProducerConsumerType.Consumer, FileArtifact = path }; return(GetProducerConsumerOfFileByKey(fileConsumerKey)); }
/// <summary> /// Ingest file and directory producer and consumer information to RocksDB /// </summary> private void IngestProducerConsumerInformation() { var parallelOptions = new ParallelOptions(); parallelOptions.MaxDegreeOfParallelism = (int)(Environment.ProcessorCount * s_innerConcurrencyMultiplier); Parallel.ForEach(m_fileConsumerMap, parallelOptions, kvp => { var fileConsumerKey = new FileProducerConsumerKey() { Type = ProducerConsumerType.Consumer, FilePath = AbsolutePathToXldbString(kvp.Key.Path), RewriteCount = kvp.Key.RewriteCount }; var fileConsumerValue = new FileConsumerValue(); fileConsumerValue.PipIds.AddRange(kvp.Value); var key = fileConsumerKey.ToByteArray(); var value = fileConsumerValue.ToByteArray(); WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.FileConsumer, key.Length + value.Length); }); Parallel.ForEach(m_directoryConsumerMap, parallelOptions, kvp => { var directoryConsumerKey = new DirectoryProducerConsumerKey() { Type = ProducerConsumerType.Consumer, DirectoryPath = AbsolutePathToXldbString(kvp.Key.Path) }; var directoryConsumerValue = new DirectoryConsumerValue(); directoryConsumerValue.PipIds.AddRange(kvp.Value); var key = directoryConsumerKey.ToByteArray(); var value = directoryConsumerValue.ToByteArray(); WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.DirectoryConsumer, key.Length + value.Length); }); Parallel.ForEach(CachedGraph.PipGraph.AllFilesAndProducers, parallelOptions, kvp => { var fileProducerKey = new FileProducerConsumerKey() { Type = ProducerConsumerType.Producer, FilePath = AbsolutePathToXldbString(kvp.Key.Path), RewriteCount = kvp.Key.RewriteCount }; var fileProducerValue = new FileProducerValue() { PipId = kvp.Value.Value }; var key = fileProducerKey.ToByteArray(); var value = fileProducerValue.ToByteArray(); WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.FileProducer, key.Length + value.Length); }); Parallel.ForEach(CachedGraph.PipGraph.AllOutputDirectoriesAndProducers, parallelOptions, kvp => { var directoryProducerKey = new DirectoryProducerConsumerKey() { Type = ProducerConsumerType.Producer, DirectoryPath = AbsolutePathToXldbString(kvp.Key.Path) }; var directoryProducerValue = new DirectoryProducerValue() { PipId = kvp.Value.Value }; var key = directoryProducerKey.ToByteArray(); var value = directoryProducerValue.ToByteArray(); WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.DirectoryProducer, key.Length + value.Length); }); Parallel.ForEach(m_dynamicFileProducerMap, parallelOptions, kvp => { var fileProducerKey = new FileProducerConsumerKey() { Type = ProducerConsumerType.Producer, FilePath = AbsolutePathToXldbString(kvp.Key.Path), RewriteCount = kvp.Key.RewriteCount }; var fileProducerValue = new FileProducerValue() { PipId = kvp.Value }; var key = fileProducerKey.ToByteArray(); var value = fileProducerValue.ToByteArray(); WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.FileProducer, key.Length + value.Length); }); }