/// <summary> /// Gets producers or consumers of a directory based on the key passed in /// </summary> private IEnumerable <uint> GetProducerConsumerOfDirectoryByKey(DirectoryProducerConsumerKey key) { Contract.Requires(Accessor != null, "XldbDataStore is not initialized"); var directoryProducerOrConsumers = new List <uint>(); var maybeFound = Accessor.Use(database => { foreach (var kvp in database.PrefixSearch(key.ToByteArray(), StaticGraphColumnFamilyName)) { if (key.Type == ProducerConsumerType.Producer) { var pipId = FileProducerValue.Parser.ParseFrom(kvp.Value).PipId; directoryProducerOrConsumers.Add(pipId); } else if (key.Type == ProducerConsumerType.Consumer) { directoryProducerOrConsumers.AddRange(FileConsumerValue.Parser.ParseFrom(kvp.Value).PipIds); } } }); if (!maybeFound.Succeeded) { maybeFound.Failure.Throw(); } return(directoryProducerOrConsumers); }
/// <summary> /// Ingest file and directory producer and consumer information to RocksDB /// </summary> private void IngestProducerConsumerInformation() { var parallelOptions = new ParallelOptions(); parallelOptions.MaxDegreeOfParallelism = (int)(Environment.ProcessorCount * s_innerConcurrencyMultiplier); Parallel.ForEach(m_fileConsumerMap, parallelOptions, kvp => { var fileConsumerKey = new FileProducerConsumerKey() { Type = ProducerConsumerType.Consumer, FilePath = AbsolutePathToXldbString(kvp.Key.Path), RewriteCount = kvp.Key.RewriteCount }; var fileConsumerValue = new FileConsumerValue(); fileConsumerValue.PipIds.AddRange(kvp.Value); var key = fileConsumerKey.ToByteArray(); var value = fileConsumerValue.ToByteArray(); WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.FileConsumer, key.Length + value.Length); }); Parallel.ForEach(m_directoryConsumerMap, parallelOptions, kvp => { var directoryConsumerKey = new DirectoryProducerConsumerKey() { Type = ProducerConsumerType.Consumer, DirectoryPath = AbsolutePathToXldbString(kvp.Key.Path) }; var directoryConsumerValue = new DirectoryConsumerValue(); directoryConsumerValue.PipIds.AddRange(kvp.Value); var key = directoryConsumerKey.ToByteArray(); var value = directoryConsumerValue.ToByteArray(); WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.DirectoryConsumer, key.Length + value.Length); }); Parallel.ForEach(CachedGraph.PipGraph.AllFilesAndProducers, parallelOptions, kvp => { var fileProducerKey = new FileProducerConsumerKey() { Type = ProducerConsumerType.Producer, FilePath = AbsolutePathToXldbString(kvp.Key.Path), RewriteCount = kvp.Key.RewriteCount }; var fileProducerValue = new FileProducerValue() { PipId = kvp.Value.Value }; var key = fileProducerKey.ToByteArray(); var value = fileProducerValue.ToByteArray(); WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.FileProducer, key.Length + value.Length); }); Parallel.ForEach(CachedGraph.PipGraph.AllOutputDirectoriesAndProducers, parallelOptions, kvp => { var directoryProducerKey = new DirectoryProducerConsumerKey() { Type = ProducerConsumerType.Producer, DirectoryPath = AbsolutePathToXldbString(kvp.Key.Path) }; var directoryProducerValue = new DirectoryProducerValue() { PipId = kvp.Value.Value }; var key = directoryProducerKey.ToByteArray(); var value = directoryProducerValue.ToByteArray(); WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.DirectoryProducer, key.Length + value.Length); }); Parallel.ForEach(m_dynamicFileProducerMap, parallelOptions, kvp => { var fileProducerKey = new FileProducerConsumerKey() { Type = ProducerConsumerType.Producer, FilePath = AbsolutePathToXldbString(kvp.Key.Path), RewriteCount = kvp.Key.RewriteCount }; var fileProducerValue = new FileProducerValue() { PipId = kvp.Value }; var key = fileProducerKey.ToByteArray(); var value = fileProducerValue.ToByteArray(); WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName); AddToDbStorageDictionary(DBStoredTypes.FileProducer, key.Length + value.Length); }); }