internal List <Folder> GetAllWatchedFolders()
{
if (watchedFoldersMap.Any())
{
return(watchedFoldersMap.Select(kv => kv.Value).ToList());
}
return(new List <Folder>());
}
private void ProcessDataFromWorkers(BPlusTree <string, int> map)
{
using (var context = NetMQContext.Create())
{
using (NetMQSocket receiver = context.CreatePullSocket())
{
Task.Run(() =>
{
// The Sink listens to the Ventilator for news about jobs
using (NetMQSocket sink = context.CreatePullSocket())
{
sink.Connect(_ventilatorReceiveAddress);
while (_writingData)
{
string message = "";
try
{
message = sink.ReceiveString();
}
catch (ObjectDisposedException e)
{
// We're done. Exit the loop.
break;
}
// A new job was pushed
if (message != "done")
{
_knownJobs.Add(message);
}
// We're done!
else
{
Task.Run(() =>
{
// While the count isn't the same, wait.
// Todo: Bake better status monitoring to prevent deadlocking
// if things are ever dropped.
while (_processedJobs.Count != _knownJobs.Count)
{
Thread.Sleep(10);
}
receiver.Close();
sink.Close();
_writingData = false;
});
}
}
}
});
receiver.Bind(_receiverAddress);
var dict = new Dictionary <string, int>();
while (_writingData)
{
string message = "";
try
{
message = receiver.ReceiveString();
}
catch (ObjectDisposedException e)
{
// We know there isn't more work, so we break this loop.
break;
}
var jobResult = JsonConvert.DeserializeObject <JobResult>(message);
foreach (var word in jobResult.Data)
{
// Update existing values
if (dict.ContainsKey(word.Key))
{
dict[word.Key] = word.Value + dict[word.Key];
}
else // Add new value
{
dict[word.Key] = word.Value;
}
// This is around the max that we can hold in 4gb safely.
// Assuming our strings aren't suuuper long.
if (dict.Count > 3020000)
{
Console.WriteLine("Flushing " + jobResult.JobId + "... ");
var stopwatch = new Stopwatch();
stopwatch.Start();
FlushToDisk(map, dict);
dict = new Dictionary <string, int>();
stopwatch.Stop();
Console.WriteLine("Done flushing " + jobResult.JobId + ". Time: " + stopwatch.ElapsedMilliseconds + "ms");
}
}
_processedJobs.Add(jobResult.JobId);
}
Console.WriteLine("Flattening data to TSV file.");
// Save our changes to the ondisk db
if (map.Any())
{
Console.WriteLine("Pushing data to BPlusTree.");
FlushToDisk(map, dict);
Console.WriteLine(map.Count);
WriteToTSVFile(map, "Output.tsv");
}
// Everything is in memory so just use that.
else
{
WriteToTSVFile(dict, "Output.tsv");
}
// We're done!
WaitingOnWork = false;
}
}
}
