/// <summary> /// Constructor /// </summary> /// <param name="opt"></param> public Pipeline(IngestOptions opt) { options = opt; status = Status.AllGood; // Threads conqueCSV = new ConcurrentQueue <string>(); flagCSV = false; conqueFilter = new ConcurrentQueue <string>(); flagFiltered = false; conqueDictionarized = new ConcurrentQueue <int[]>(); flagDictionarized = false; // Dictionarizing workingMasterDictionary = new ConcurrentQueue <string>(); workingMasterWordCloud = new ConcurrentDictionary <string, int>(); // Markovization workingMasterNGramCloud = new ConcurrentDictionary <NGram, int>(); workingMasterNGrams = new ConcurrentQueue <NGram>(); workingMasterSeeds = new ConcurrentDictionary <int, bool>(); workingMasterSuccessors = new ConcurrentDictionary <int, ConcurrentDictionary <int, int> >(); }
public PostIngestPipe(IngestOptions opt, Ingester filter_previ, ConcurrentQueue <string> msgs) { inMessages = msgs; localFilter = new Filter(opt.regexFilters, msgs); localDic = new Dictionarizer(localFilter.PieceTask, localFilter.outMessageStrings); localMark = new Markovizer(opt.gramSize, localDic.PieceTask, localDic.outSentenceBank); }
public MarkovPipe(IngestOptions opts) { ingestOptions = opts; localIngester = new Ingester(ingestOptions.infileCSV); workingPostIngestPipes = new Dictionary <ulong, PostIngestPipe>(); workingPostIngestInMsgs = new Dictionary <ulong, ConcurrentQueue <string> >(); Names = new Dictionary <ulong, string>(); }