/// <summary>
/// Constructor
/// </summary>
/// <param name="opt"></param>
public Pipeline(IngestOptions opt)
{
options = opt;
status = Status.AllGood;
// Threads
conqueCSV = new ConcurrentQueue <string>();
flagCSV = false;
conqueFilter = new ConcurrentQueue <string>();
flagFiltered = false;
conqueDictionarized = new ConcurrentQueue <int[]>();
flagDictionarized = false;
// Dictionarizing
workingMasterDictionary = new ConcurrentQueue <string>();
workingMasterWordCloud = new ConcurrentDictionary <string, int>();
// Markovization
workingMasterNGramCloud = new ConcurrentDictionary <NGram, int>();
workingMasterNGrams = new ConcurrentQueue <NGram>();
workingMasterSeeds = new ConcurrentDictionary <int, bool>();
workingMasterSuccessors = new ConcurrentDictionary <int, ConcurrentDictionary <int, int> >();
}
public PostIngestPipe(IngestOptions opt, Ingester filter_previ, ConcurrentQueue <string> msgs)
{
inMessages = msgs;
localFilter = new Filter(opt.regexFilters, msgs);
localDic = new Dictionarizer(localFilter.PieceTask, localFilter.outMessageStrings);
localMark = new Markovizer(opt.gramSize, localDic.PieceTask, localDic.outSentenceBank);
}
public MarkovPipe(IngestOptions opts)
{
ingestOptions = opts;
localIngester = new Ingester(ingestOptions.infileCSV);
workingPostIngestPipes = new Dictionary <ulong, PostIngestPipe>();
workingPostIngestInMsgs = new Dictionary <ulong, ConcurrentQueue <string> >();
Names = new Dictionary <ulong, string>();
}
