/// <summary>
/// Leave only new works.
/// </summary>
/// <param name="syncronizer"></param>
public void FilterOnlyNewed(Syncronizer syncronizer)
{
var hitomi_ids = new HashSet<int>();
syncronizer.newedDataHitomi.ForEach(x => hitomi_ids.Add(x));
var eh_ids = new HashSet<int>();
syncronizer.newedDataEH.ForEach(x => eh_ids.Add(x));
HitomiData.Instance.metadata_collection.RemoveAll(x => !hitomi_ids.Contains(x.ID));
ehentaiArticles.RemoveAll(x => !eh_ids.Contains(int.Parse(x.URL.Split('/')[4])));
// You donot have to remove ehIndex's items.
}
static void ProcessStart(bool include_exhentai, bool low_perf, bool sync_only, bool use_server, bool use_elasticsearch,
string[] hitomi_sync_range, string[] hitomi_sync_lookup_range, string[] exhentai_lookup_page, bool sync_only_hitomi)
{
Console.Clear();
Console.Title = "hsync";
Console.WriteLine($"hsync - DB Synchronization Manager");
Console.WriteLine($"Copyright (C) 2020. project violet-server.");
Console.WriteLine($"Version: {Version.Text} (Build: {Internals.GetBuildDate().ToLongDateString()})");
Console.WriteLine("");
if (!low_perf)
{
if (!File.Exists("hiddendata.json"))
{
Logs.Instance.Push("Welcome to hsync!\r\n\tDownload the necessary data before running the program!");
download_data("https://github.com/project-violet/database/releases/download/rd2020.06.07/hiddendata.json", "hiddendata.json");
}
if (!File.Exists("metadata.json"))
{
download_data("https://github.com/project-violet/database/releases/download/rd2020.06.07/metadata.json", "metadata.json");
}
if (!File.Exists("ex-hentai-archive.json"))
{
download_data("https://github.com/project-violet/database/releases/download/rd2020.06.07/ex-hentai-archive.json", "ex-hentai-archive.json");
}
var sync = new Syncronizer(hitomi_sync_range, hitomi_sync_lookup_range, exhentai_lookup_page);
sync.SyncHitomi();
if (!sync_only_hitomi)
{
sync.SyncExHentai();
}
if (sync_only)
{
return;
}
var dbc = new DataBaseCreator();
dbc.Integrate();
dbc.ExtractRawDatabase("rawdata", include_exhentai: include_exhentai);
Console.WriteLine("Complete all!");
dbc.ExtractRawDatabase("rawdata-chinese", false, "chinese", include_exhentai: include_exhentai);
Console.WriteLine("Complete chinese!");
dbc.ExtractRawDatabase("rawdata-english", false, "english", include_exhentai: include_exhentai);
Console.WriteLine("Complete english!");
dbc.ExtractRawDatabase("rawdata-japanese", false, "japanese", include_exhentai: include_exhentai);
Console.WriteLine("Complete japanese!");
dbc.ExtractRawDatabase("rawdata-korean", false, "korean", include_exhentai: include_exhentai);
Console.WriteLine("Complete korean!");
//dbc.FilterOnlyNewed(sync);
//dbc.Integrate();
//var dt = DateTime.Now.ToString("yyyy-MM-dd hh-mm");
//dbc.ExtractRawDatabase($"chunk/{dt}/rawdata", true);
//Console.WriteLine("Complete all!");
//dbc.ExtractRawDatabase($"chunk/{dt}/rawdata-chinese", true, "chinese");
//Console.WriteLine("Complete chinese!");
//dbc.ExtractRawDatabase($"chunk/{dt}/rawdata-english", true, "english");
//Console.WriteLine("Complete english!");
//dbc.ExtractRawDatabase($"chunk/{dt}/rawdata-japanese", true, "japanese");
//Console.WriteLine("Complete japanese!");
//dbc.ExtractRawDatabase($"chunk/{dt}/rawdata-korean", true, "korean");
//Console.WriteLine("Complete korean!");
}
else
{
var sync = new SyncronizerLowPerf(hitomi_sync_range, hitomi_sync_lookup_range, exhentai_lookup_page);
sync.SyncHitomi();
if (!sync_only_hitomi)
{
sync.SyncExHentai();
}
sync.FlushToMainDatabase();
if (use_server)
{
sync.FlushToServerDatabase();
}
if (use_elasticsearch)
{
sync.FlushToElasticSearchServer();
}
if (sync_only)
{
return;
}
var dbc = new DataBaseCreatorLowPerf();
dbc.ExtractRawDatabase("rawdata", include_exhentai: include_exhentai);
Console.WriteLine("Complete all!");
dbc.ExtractRawDatabase("rawdata-chinese", "chinese", include_exhentai: include_exhentai);
Console.WriteLine("Complete chinese!");
dbc.ExtractRawDatabase("rawdata-english", "english", include_exhentai: include_exhentai);
Console.WriteLine("Complete english!");
dbc.ExtractRawDatabase("rawdata-japanese", "japanese", include_exhentai: include_exhentai);
Console.WriteLine("Complete japanese!");
dbc.ExtractRawDatabase("rawdata-korean", "korean", include_exhentai: include_exhentai);
Console.WriteLine("Complete korean!");
}
}