private void ConcurrentWorker(Object state)
{
var data = new DataImporter();
HashSet<string> commentSubs = data.fetchCommentSubmissions();
int i = 0, j = 0; //Counter for sumbissions and ignores
string line;
bool successfulTake = false;
Stopwatch startWatch = new Stopwatch(), loopWatch = new Stopwatch();
startWatch.Start();
loopWatch.Start();
while (!endOfFile) {
while (!lineBag.IsEmpty) {
successfulTake = lineBag.TryTake(out line);
if (successfulTake) {
Submission sub = JsonConvert.DeserializeObject<Submission>(line, SubmissionResolver.jSetIdOnly);
if (commentSubs.Any(l => l.EndsWith(sub.id))) {
//Load all of the fields in for this line
Submission subAll = JsonConvert.DeserializeObject<Submission>(line, SubmissionResolver.jSetAll);
Client2.Entities.PostAsync(subAll);
commentSubs.RemoveWhere(l => l.EndsWith(subAll.id));
i++;
}
else {
j++;
if (j % 100000 == 0) {
//Console.Clear();
Console.WriteLine("Stats\nProcessed: {0}", j);
Console.WriteLine("Found: {0}", i);
Console.WriteLine("Loop Time: {0}", loopWatch.Elapsed);
Console.WriteLine("Total Time: {0}", startWatch.Elapsed);
loopWatch.Restart();
}
}
}
}
logger.Info("Linebag is empty but not at EOF yet, sleeping.");
Thread.Sleep(workerSleepTime);
}
logger.Info("Finished processing, there was " + i + " new submissions added to the db and " + j + " submissions ignored");
threadsToComplete--;
}
static void Main(string[] args)
{
BasicConfigurator.Configure();
while (true) {
Console.Write("Choose an operation: ");
string action = Console.ReadLine();
switch (action) {
case "0":
return;
case "1": {
var fs = new FileStream(CORPUS_PATH, FileMode.Open, FileAccess.Read, FileShare.Read);
Program p = new Program(fs);
p.buildDomainStats();
break;
}
case "2": {
Program p = new Program();
Console.Write("Submission ID: ");
p.fetchSubmission(Console.ReadLine());
break;
}
case "3": {
var fs = new FileStream(COMMENT_08, FileMode.Open, FileAccess.Read, FileShare.Read);
DataImporter di = new DataImporter();
di.ReadInComment(fs, Program.Client1, 100);
break;
}
default:
Console.WriteLine("Unknown opertation");
break;
}
}
//Data load in thread
//ThreadPool.QueueUserWorkItem(new WaitCallback(p.loadConcurrent));
//for (int i = 0; i < 5; i++) {
// ThreadPool.QueueUserWorkItem(new WaitCallback(p.ConcurrentWorker));
// p.threadsToComplete++;
//}
//while (p.threadsToComplete > 0) {
// Thread.Sleep(12000);
// Console.Clear();
//}
//Console.WriteLine("All comment size:" + p.allComments.Count);
//FileStream fs = File.OpenRead(CORPUS_PATH);
//fs.Seek(8688, 0);
//for (int i = 0; i < 100; i++) {
// Console.Write("{0}", (char)fs.ReadByte());
//}
//var indexer = new Indexer();
////indexer.buildIndex(fs).Wait();
////indexer.verifyIndex(fs, 1378479584);
}
