/// <summary>
/// Gets producers or consumers of a directory based on the key passed in
/// </summary>
private IEnumerable <uint> GetProducerConsumerOfDirectoryByKey(DirectoryProducerConsumerKey key)
{
Contract.Requires(Accessor != null, "XldbDataStore is not initialized");
var directoryProducerOrConsumers = new List <uint>();
var maybeFound = Accessor.Use(database =>
{
foreach (var kvp in database.PrefixSearch(key.ToByteArray(), StaticGraphColumnFamilyName))
{
if (key.Type == ProducerConsumerType.Producer)
{
var pipId = FileProducerValue.Parser.ParseFrom(kvp.Value).PipId;
directoryProducerOrConsumers.Add(pipId);
}
else if (key.Type == ProducerConsumerType.Consumer)
{
directoryProducerOrConsumers.AddRange(FileConsumerValue.Parser.ParseFrom(kvp.Value).PipIds);
}
}
});
if (!maybeFound.Succeeded)
{
maybeFound.Failure.Throw();
}
return(directoryProducerOrConsumers);
}
/// <summary>
/// Gets all consumers of a particular directory
/// </summary>
public IEnumerable <uint> GetConsumersOfDirectory(string path)
{
Contract.Requires(Accessor != null, "XldbDataStore is not initialized");
var directoryConsumerKey = new DirectoryProducerConsumerKey()
{
Type = ProducerConsumerType.Consumer,
DirectoryArtifact = path
};
return(GetProducerConsumerOfDirectoryByKey(directoryConsumerKey));
}
/// <summary>
/// Ingest file and directory producer and consumer information to RocksDB
/// </summary>
private void IngestProducerConsumerInformation()
{
var parallelOptions = new ParallelOptions();
parallelOptions.MaxDegreeOfParallelism = (int)(Environment.ProcessorCount * s_innerConcurrencyMultiplier);
Parallel.ForEach(m_fileConsumerMap, parallelOptions, kvp =>
{
var fileConsumerKey = new FileProducerConsumerKey()
{
Type = ProducerConsumerType.Consumer,
FilePath = AbsolutePathToXldbString(kvp.Key.Path),
RewriteCount = kvp.Key.RewriteCount
};
var fileConsumerValue = new FileConsumerValue();
fileConsumerValue.PipIds.AddRange(kvp.Value);
var key = fileConsumerKey.ToByteArray();
var value = fileConsumerValue.ToByteArray();
WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName);
AddToDbStorageDictionary(DBStoredTypes.FileConsumer, key.Length + value.Length);
});
Parallel.ForEach(m_directoryConsumerMap, parallelOptions, kvp =>
{
var directoryConsumerKey = new DirectoryProducerConsumerKey()
{
Type = ProducerConsumerType.Consumer,
DirectoryPath = AbsolutePathToXldbString(kvp.Key.Path)
};
var directoryConsumerValue = new DirectoryConsumerValue();
directoryConsumerValue.PipIds.AddRange(kvp.Value);
var key = directoryConsumerKey.ToByteArray();
var value = directoryConsumerValue.ToByteArray();
WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName);
AddToDbStorageDictionary(DBStoredTypes.DirectoryConsumer, key.Length + value.Length);
});
Parallel.ForEach(CachedGraph.PipGraph.AllFilesAndProducers, parallelOptions, kvp =>
{
var fileProducerKey = new FileProducerConsumerKey()
{
Type = ProducerConsumerType.Producer,
FilePath = AbsolutePathToXldbString(kvp.Key.Path),
RewriteCount = kvp.Key.RewriteCount
};
var fileProducerValue = new FileProducerValue()
{
PipId = kvp.Value.Value
};
var key = fileProducerKey.ToByteArray();
var value = fileProducerValue.ToByteArray();
WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName);
AddToDbStorageDictionary(DBStoredTypes.FileProducer, key.Length + value.Length);
});
Parallel.ForEach(CachedGraph.PipGraph.AllOutputDirectoriesAndProducers, parallelOptions, kvp =>
{
var directoryProducerKey = new DirectoryProducerConsumerKey()
{
Type = ProducerConsumerType.Producer,
DirectoryPath = AbsolutePathToXldbString(kvp.Key.Path)
};
var directoryProducerValue = new DirectoryProducerValue()
{
PipId = kvp.Value.Value
};
var key = directoryProducerKey.ToByteArray();
var value = directoryProducerValue.ToByteArray();
WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName);
AddToDbStorageDictionary(DBStoredTypes.DirectoryProducer, key.Length + value.Length);
});
Parallel.ForEach(m_dynamicFileProducerMap, parallelOptions, kvp =>
{
var fileProducerKey = new FileProducerConsumerKey()
{
Type = ProducerConsumerType.Producer,
FilePath = AbsolutePathToXldbString(kvp.Key.Path),
RewriteCount = kvp.Key.RewriteCount
};
var fileProducerValue = new FileProducerValue()
{
PipId = kvp.Value
};
var key = fileProducerKey.ToByteArray();
var value = fileProducerValue.ToByteArray();
WriteToDb(key, value, XldbDataStore.StaticGraphColumnFamilyName);
AddToDbStorageDictionary(DBStoredTypes.FileProducer, key.Length + value.Length);
});
}
