private void ConcurrentWorker(Object state) { var data = new DataImporter(); HashSet<string> commentSubs = data.fetchCommentSubmissions(); int i = 0, j = 0; //Counter for sumbissions and ignores string line; bool successfulTake = false; Stopwatch startWatch = new Stopwatch(), loopWatch = new Stopwatch(); startWatch.Start(); loopWatch.Start(); while (!endOfFile) { while (!lineBag.IsEmpty) { successfulTake = lineBag.TryTake(out line); if (successfulTake) { Submission sub = JsonConvert.DeserializeObject<Submission>(line, SubmissionResolver.jSetIdOnly); if (commentSubs.Any(l => l.EndsWith(sub.id))) { //Load all of the fields in for this line Submission subAll = JsonConvert.DeserializeObject<Submission>(line, SubmissionResolver.jSetAll); Client2.Entities.PostAsync(subAll); commentSubs.RemoveWhere(l => l.EndsWith(subAll.id)); i++; } else { j++; if (j % 100000 == 0) { //Console.Clear(); Console.WriteLine("Stats\nProcessed: {0}", j); Console.WriteLine("Found: {0}", i); Console.WriteLine("Loop Time: {0}", loopWatch.Elapsed); Console.WriteLine("Total Time: {0}", startWatch.Elapsed); loopWatch.Restart(); } } } } logger.Info("Linebag is empty but not at EOF yet, sleeping."); Thread.Sleep(workerSleepTime); } logger.Info("Finished processing, there was " + i + " new submissions added to the db and " + j + " submissions ignored"); threadsToComplete--; }
static void Main(string[] args) { BasicConfigurator.Configure(); while (true) { Console.Write("Choose an operation: "); string action = Console.ReadLine(); switch (action) { case "0": return; case "1": { var fs = new FileStream(CORPUS_PATH, FileMode.Open, FileAccess.Read, FileShare.Read); Program p = new Program(fs); p.buildDomainStats(); break; } case "2": { Program p = new Program(); Console.Write("Submission ID: "); p.fetchSubmission(Console.ReadLine()); break; } case "3": { var fs = new FileStream(COMMENT_08, FileMode.Open, FileAccess.Read, FileShare.Read); DataImporter di = new DataImporter(); di.ReadInComment(fs, Program.Client1, 100); break; } default: Console.WriteLine("Unknown opertation"); break; } } //Data load in thread //ThreadPool.QueueUserWorkItem(new WaitCallback(p.loadConcurrent)); //for (int i = 0; i < 5; i++) { // ThreadPool.QueueUserWorkItem(new WaitCallback(p.ConcurrentWorker)); // p.threadsToComplete++; //} //while (p.threadsToComplete > 0) { // Thread.Sleep(12000); // Console.Clear(); //} //Console.WriteLine("All comment size:" + p.allComments.Count); //FileStream fs = File.OpenRead(CORPUS_PATH); //fs.Seek(8688, 0); //for (int i = 0; i < 100; i++) { // Console.Write("{0}", (char)fs.ReadByte()); //} //var indexer = new Indexer(); ////indexer.buildIndex(fs).Wait(); ////indexer.verifyIndex(fs, 1378479584); }