public void Start()
{
Initialize();
max = tags_list.Count;
progress = 0;
mtl = Environment.ProcessorCount;
Console.Write("Process... ");
using (var pb = new ExtractingProgressBar())
{
Task.WhenAll(Enumerable.Range(0, mtl).Select(no => Task.Run(() => process(no, pb)))).Wait();
}
Console.WriteLine("Complete");
Console.Write("Merge... ");
Merge();
Console.WriteLine("Complete");
Console.Write("Save... ");
var rr = result.ToList();
rr.Sort((x, y) => y.Value.Count.CompareTo(x.Value.Count));
JArray arr = new JArray();
rr.ForEach(x =>
{
JArray tags = new JArray();
x.Value.ForEach(y =>
{
tags.Add(new JObject {
{ y.Item1, y.Item2 }
});
});
arr.Add(new JObject {
{ x.Key, tags }
});
});
File.WriteAllText(Path.Combine(dbdir, "ct-result.json"), JsonConvert.SerializeObject(arr, Formatting.Indented));
Console.WriteLine("Complete");
}
private void process(int i, ExtractingProgressBar pb)
{
int min = this.max / mtl * i;
int max = this.max / mtl * (i + 1);
if (max > this.max)
{
max = this.max;
}
List <Tuple <string, string, double> > result = new List <Tuple <string, string, double> >();
for (int j = max - 1; j >= min; j--)
{
result.AddRange(Intersect(j));
pb.Report(this.max, Interlocked.Increment(ref progress));
}
lock (results)
results.AddRange(result);
}
static void ProcessTest(string[] args)
{
switch (args[0])
{
case "help":
Console.WriteLine("");
break;
case "latestrows":
var db = new SQLiteConnection("data.db");
var rr = db.Query <HitomiColumnModel>("SELECT * FROM HitomiColumnModel WHERE Language='korean'" +
" ORDER BY Id DESC LIMIT 10", new object[] { });
foreach (var r in rr)
{
Console.WriteLine(JsonConvert.SerializeObject(r));
}
break;
case "latestexcomment":
{
var record = JsonConvert.DeserializeObject <Dictionary <int, List <Tuple <DateTime, string, string> > > >(File.ReadAllText("excomment-zip.json"));
Console.WriteLine(record.ToList().Max(x => x.Key));
}
break;
case "excommentzip":
{
var comment_files = Directory.GetFiles("./ex");
var articles = new Dictionary <int, List <Tuple <DateTime, string, string> > >();
var authors = new Dictionary <string, int>();
using (var pb = new ExtractingProgressBar())
{
int count = 1;
foreach (var file in comment_files)
{
if (!file.EndsWith(".json"))
{
continue;
}
var comments = JsonConvert.DeserializeObject <List <Tuple <DateTime, string, string> > >(File.ReadAllText(file));
articles.Add(int.Parse(Path.GetFileNameWithoutExtension(file)), comments);
comments.ForEach(x =>
{
if (!authors.ContainsKey(x.Item2))
{
authors.Add(x.Item2, 0);
}
authors[x.Item2] += 1;
});
pb.Report(comment_files.Length, count++);
}
}
Console.WriteLine("Total Comments: " + articles.ToList().Sum(x => x.Value.Count));
var ll = articles.ToList();
ll.Sort((x, y) => y.Value.Count.CompareTo(x.Value.Count));
Console.WriteLine("Most Commented Articles: \r\n" + string.Join("\r\n", ll.Take(50).Select(x => $"{x.Key} ({x.Value.Count})")));
var ll2 = authors.ToList();
ll2.Sort((x, y) => y.Value.CompareTo(x.Value));
Console.WriteLine("Most Commented Authors: \r\n" + string.Join("\r\n", ll2.Take(50).Select(x => $"{x.Key} ({x.Value})")));
var record = JsonConvert.DeserializeObject <Dictionary <int, List <Tuple <DateTime, string, string> > > >(File.ReadAllText("excomment-zip.json"));
var rll = record.ToList();
rll.ForEach(x =>
{
if (!articles.ContainsKey(x.Key))
{
articles.Add(x.Key, x.Value);
}
});
File.WriteAllText("excomment-zip.json", JsonConvert.SerializeObject(articles, Formatting.Indented));
}
break;
case "excommentsearch":
{
var articles =
JsonConvert.DeserializeObject <Dictionary <int, List <Tuple <DateTime, string, string> > > >(File.ReadAllText("excomment-zip.json"));
var ll = articles.ToList();
ll.Sort((x, y) => x.Key.CompareTo(y.Key));
var x = string.Join("\r\n---------------------------------------\r\n",
ll.Select(x => string.Join("\r\n", x.Value.Where(x =>
x.Item3.Contains("dcinside")).Select(y => $"({x.Key}) [{y.Item2}] {y.Item3}"))).Where(x => x.Length > 0));
Console.WriteLine(x);
}
break;
}
}
