public void Start() { Initialize(); max = tags_list.Count; progress = 0; mtl = Environment.ProcessorCount; Console.Write("Process... "); using (var pb = new ExtractingProgressBar()) { Task.WhenAll(Enumerable.Range(0, mtl).Select(no => Task.Run(() => process(no, pb)))).Wait(); } Console.WriteLine("Complete"); Console.Write("Merge... "); Merge(); Console.WriteLine("Complete"); Console.Write("Save... "); var rr = result.ToList(); rr.Sort((x, y) => y.Value.Count.CompareTo(x.Value.Count)); JArray arr = new JArray(); rr.ForEach(x => { JArray tags = new JArray(); x.Value.ForEach(y => { tags.Add(new JObject { { y.Item1, y.Item2 } }); }); arr.Add(new JObject { { x.Key, tags } }); }); File.WriteAllText(Path.Combine(dbdir, "ct-result.json"), JsonConvert.SerializeObject(arr, Formatting.Indented)); Console.WriteLine("Complete"); }
private void process(int i, ExtractingProgressBar pb) { int min = this.max / mtl * i; int max = this.max / mtl * (i + 1); if (max > this.max) { max = this.max; } List <Tuple <string, string, double> > result = new List <Tuple <string, string, double> >(); for (int j = max - 1; j >= min; j--) { result.AddRange(Intersect(j)); pb.Report(this.max, Interlocked.Increment(ref progress)); } lock (results) results.AddRange(result); }
static void ProcessTest(string[] args) { switch (args[0]) { case "help": Console.WriteLine(""); break; case "latestrows": var db = new SQLiteConnection("data.db"); var rr = db.Query <HitomiColumnModel>("SELECT * FROM HitomiColumnModel WHERE Language='korean'" + " ORDER BY Id DESC LIMIT 10", new object[] { }); foreach (var r in rr) { Console.WriteLine(JsonConvert.SerializeObject(r)); } break; case "latestexcomment": { var record = JsonConvert.DeserializeObject <Dictionary <int, List <Tuple <DateTime, string, string> > > >(File.ReadAllText("excomment-zip.json")); Console.WriteLine(record.ToList().Max(x => x.Key)); } break; case "excommentzip": { var comment_files = Directory.GetFiles("./ex"); var articles = new Dictionary <int, List <Tuple <DateTime, string, string> > >(); var authors = new Dictionary <string, int>(); using (var pb = new ExtractingProgressBar()) { int count = 1; foreach (var file in comment_files) { if (!file.EndsWith(".json")) { continue; } var comments = JsonConvert.DeserializeObject <List <Tuple <DateTime, string, string> > >(File.ReadAllText(file)); articles.Add(int.Parse(Path.GetFileNameWithoutExtension(file)), comments); comments.ForEach(x => { if (!authors.ContainsKey(x.Item2)) { authors.Add(x.Item2, 0); } authors[x.Item2] += 1; }); pb.Report(comment_files.Length, count++); } } Console.WriteLine("Total Comments: " + articles.ToList().Sum(x => x.Value.Count)); var ll = articles.ToList(); ll.Sort((x, y) => y.Value.Count.CompareTo(x.Value.Count)); Console.WriteLine("Most Commented Articles: \r\n" + string.Join("\r\n", ll.Take(50).Select(x => $"{x.Key} ({x.Value.Count})"))); var ll2 = authors.ToList(); ll2.Sort((x, y) => y.Value.CompareTo(x.Value)); Console.WriteLine("Most Commented Authors: \r\n" + string.Join("\r\n", ll2.Take(50).Select(x => $"{x.Key} ({x.Value})"))); var record = JsonConvert.DeserializeObject <Dictionary <int, List <Tuple <DateTime, string, string> > > >(File.ReadAllText("excomment-zip.json")); var rll = record.ToList(); rll.ForEach(x => { if (!articles.ContainsKey(x.Key)) { articles.Add(x.Key, x.Value); } }); File.WriteAllText("excomment-zip.json", JsonConvert.SerializeObject(articles, Formatting.Indented)); } break; case "excommentsearch": { var articles = JsonConvert.DeserializeObject <Dictionary <int, List <Tuple <DateTime, string, string> > > >(File.ReadAllText("excomment-zip.json")); var ll = articles.ToList(); ll.Sort((x, y) => x.Key.CompareTo(y.Key)); var x = string.Join("\r\n---------------------------------------\r\n", ll.Select(x => string.Join("\r\n", x.Value.Where(x => x.Item3.Contains("dcinside")).Select(y => $"({x.Key}) [{y.Item2}] {y.Item3}"))).Where(x => x.Length > 0)); Console.WriteLine(x); } break; } }