static void ProcessCompress(bool include_exhentai, bool low_perf)
{
Console.Clear();
Console.Title = "hsync";
Console.WriteLine($"hsync - DB Synchronization Manager");
Console.WriteLine($"Copyright (C) 2020. project violet-server.");
Console.WriteLine($"Version: {Version.Text} (Build: {Internals.GetBuildDate().ToLongDateString()})");
Console.WriteLine("");
if (!low_perf)
{
var dbc = new DataBaseCreator();
dbc.Integrate();
dbc.ExtractRawDatabase("rawdata", include_exhentai: include_exhentai);
Console.WriteLine("Complete all!");
dbc.ExtractRawDatabase("rawdata-chinese", false, "chinese", include_exhentai: include_exhentai);
Console.WriteLine("Complete chinese!");
dbc.ExtractRawDatabase("rawdata-english", false, "english", include_exhentai: include_exhentai);
Console.WriteLine("Complete english!");
dbc.ExtractRawDatabase("rawdata-japanese", false, "japanese", include_exhentai: include_exhentai);
Console.WriteLine("Complete japanese!");
dbc.ExtractRawDatabase("rawdata-korean", false, "korean", include_exhentai: include_exhentai);
Console.WriteLine("Complete korean!");
}
else
{
var dbc = new DataBaseCreatorLowPerf();
dbc.ExtractRawDatabase("rawdata", include_exhentai: include_exhentai);
Console.WriteLine("Complete all!");
dbc.ExtractRawDatabase("rawdata-chinese", "chinese", include_exhentai: include_exhentai);
Console.WriteLine("Complete chinese!");
dbc.ExtractRawDatabase("rawdata-english", "english", include_exhentai: include_exhentai);
Console.WriteLine("Complete english!");
dbc.ExtractRawDatabase("rawdata-japanese", "japanese", include_exhentai: include_exhentai);
Console.WriteLine("Complete japanese!");
dbc.ExtractRawDatabase("rawdata-korean", "korean", include_exhentai: include_exhentai);
Console.WriteLine("Complete korean!");
}
}
public static void PrintVersion()
{
Console.WriteLine($"{Version.Name} {Version.Text}");
Console.WriteLine($"Build Date: " + Internals.GetBuildDate().ToLongDateString());
}
static void ProcessStart(bool include_exhentai, bool low_perf, bool sync_only, bool use_server, bool use_elasticsearch,
string[] hitomi_sync_range, string[] hitomi_sync_lookup_range, string[] exhentai_lookup_page, bool sync_only_hitomi)
{
Console.Clear();
Console.Title = "hsync";
Console.WriteLine($"hsync - DB Synchronization Manager");
Console.WriteLine($"Copyright (C) 2020. project violet-server.");
Console.WriteLine($"Version: {Version.Text} (Build: {Internals.GetBuildDate().ToLongDateString()})");
Console.WriteLine("");
if (!low_perf)
{
if (!File.Exists("hiddendata.json"))
{
Logs.Instance.Push("Welcome to hsync!\r\n\tDownload the necessary data before running the program!");
download_data("https://github.com/project-violet/database/releases/download/rd2020.06.07/hiddendata.json", "hiddendata.json");
}
if (!File.Exists("metadata.json"))
{
download_data("https://github.com/project-violet/database/releases/download/rd2020.06.07/metadata.json", "metadata.json");
}
if (!File.Exists("ex-hentai-archive.json"))
{
download_data("https://github.com/project-violet/database/releases/download/rd2020.06.07/ex-hentai-archive.json", "ex-hentai-archive.json");
}
var sync = new Syncronizer(hitomi_sync_range, hitomi_sync_lookup_range, exhentai_lookup_page);
sync.SyncHitomi();
if (!sync_only_hitomi)
{
sync.SyncExHentai();
}
if (sync_only)
{
return;
}
var dbc = new DataBaseCreator();
dbc.Integrate();
dbc.ExtractRawDatabase("rawdata", include_exhentai: include_exhentai);
Console.WriteLine("Complete all!");
dbc.ExtractRawDatabase("rawdata-chinese", false, "chinese", include_exhentai: include_exhentai);
Console.WriteLine("Complete chinese!");
dbc.ExtractRawDatabase("rawdata-english", false, "english", include_exhentai: include_exhentai);
Console.WriteLine("Complete english!");
dbc.ExtractRawDatabase("rawdata-japanese", false, "japanese", include_exhentai: include_exhentai);
Console.WriteLine("Complete japanese!");
dbc.ExtractRawDatabase("rawdata-korean", false, "korean", include_exhentai: include_exhentai);
Console.WriteLine("Complete korean!");
//dbc.FilterOnlyNewed(sync);
//dbc.Integrate();
//var dt = DateTime.Now.ToString("yyyy-MM-dd hh-mm");
//dbc.ExtractRawDatabase($"chunk/{dt}/rawdata", true);
//Console.WriteLine("Complete all!");
//dbc.ExtractRawDatabase($"chunk/{dt}/rawdata-chinese", true, "chinese");
//Console.WriteLine("Complete chinese!");
//dbc.ExtractRawDatabase($"chunk/{dt}/rawdata-english", true, "english");
//Console.WriteLine("Complete english!");
//dbc.ExtractRawDatabase($"chunk/{dt}/rawdata-japanese", true, "japanese");
//Console.WriteLine("Complete japanese!");
//dbc.ExtractRawDatabase($"chunk/{dt}/rawdata-korean", true, "korean");
//Console.WriteLine("Complete korean!");
}
else
{
var sync = new SyncronizerLowPerf(hitomi_sync_range, hitomi_sync_lookup_range, exhentai_lookup_page);
sync.SyncHitomi();
if (!sync_only_hitomi)
{
sync.SyncExHentai();
}
sync.FlushToMainDatabase();
if (use_server)
{
sync.FlushToServerDatabase();
}
if (use_elasticsearch)
{
sync.FlushToElasticSearchServer();
}
if (sync_only)
{
return;
}
var dbc = new DataBaseCreatorLowPerf();
dbc.ExtractRawDatabase("rawdata", include_exhentai: include_exhentai);
Console.WriteLine("Complete all!");
dbc.ExtractRawDatabase("rawdata-chinese", "chinese", include_exhentai: include_exhentai);
Console.WriteLine("Complete chinese!");
dbc.ExtractRawDatabase("rawdata-english", "english", include_exhentai: include_exhentai);
Console.WriteLine("Complete english!");
dbc.ExtractRawDatabase("rawdata-japanese", "japanese", include_exhentai: include_exhentai);
Console.WriteLine("Complete japanese!");
dbc.ExtractRawDatabase("rawdata-korean", "korean", include_exhentai: include_exhentai);
Console.WriteLine("Complete korean!");
}
}
static void ProcessStart()
{
Console.Clear();
Console.Title = "hsync";
Console.WriteLine($"hsync - DB Synchronization Manager");
Console.WriteLine($"Copyright (C) 2020. project violet-server.");
Console.WriteLine($"Version: {Version.Text} (Build: {Internals.GetBuildDate().ToLongDateString()})");
Console.WriteLine("");
if (!File.Exists("hiddendata.json"))
{
Logs.Instance.Push("Welcome to hsync!\r\n\tDownload the necessary data before running the program!");
download_data("https://github.com/project-violet/database/releases/download/rd2020.06.07/hiddendata.json", "hiddendata.json");
}
if (!File.Exists("metadata.json"))
{
download_data("https://github.com/project-violet/database/releases/download/rd2020.06.07/metadata.json", "metadata.json");
}
if (!File.Exists("ex-hentai-archive.json"))
{
download_data("https://github.com/project-violet/database/releases/download/rd2020.06.07/ex-hentai-archive.json", "ex-hentai-archive.json");
}
HitomiData.Instance.Load();
var latest = HitomiData.Instance.metadata_collection.First().ID;
//#if true
// Sync Hitomi
{
var range = 2000;
var exists = new HashSet <int>();
foreach (var metadata in HitomiData.Instance.metadata_collection)
{
exists.Add(metadata.ID);
}
var gburls = Enumerable.Range(latest - range, range * 2).Where(x => !exists.Contains(x)).Select(x => $"https://ltn.hitomi.la/galleryblock/{x}.html").ToList();
var dcnt = 0;
var ecnt = 0;
Console.Write("Running galleryblock tester... ");
List <string> htmls;
using (var pb = new ProgressBar())
{
htmls = NetTools.DownloadStrings(gburls, "",
() => {
pb.Report(gburls.Count, Interlocked.Increment(ref dcnt), ecnt);
},
() => {
pb.Report(gburls.Count, dcnt, Interlocked.Increment(ref ecnt));
});
}
Console.WriteLine("Complete");
var gurls = new List <string>(gburls.Count);
for (int i = 0; i < gburls.Count; i++)
{
if (htmls[i] == null)
{
continue;
}
var aa = HitomiParser.ParseGalleryBlock(htmls[i]);
if (aa.Magic.Contains("-"))
{
gurls.Add("https://hitomi.la/" + aa.Magic);
}
else
{
gurls.Add("https://hitomi.la/galleries/" + i + ".html");
}
}
dcnt = 0;
ecnt = 0;
Console.Write("Running gallery tester... ");
List <string> htmls2 = null;
if (gurls.Count != 0)
{
using (var pb = new ProgressBar())
{
htmls2 = NetTools.DownloadStrings(gurls, "",
() => {
pb.Report(gburls.Count, Interlocked.Increment(ref dcnt), ecnt);
},
() => {
pb.Report(gburls.Count, dcnt, Interlocked.Increment(ref ecnt));
});
}
}
Console.WriteLine("Complete");
Console.Write("Check redirect gallery html... ");
var last_change = true;
while (last_change)
{
last_change = false;
for (int i = 0; i < htmls2.Count; i++)
{
if (htmls2[i] == null)
{
continue;
}
var node = htmls2[i].ToHtmlNode();
var title = node.SelectSingleNode("//title");
if (title != null && title.InnerText == "Redirect")
{
htmls2[i] = NetTools.DownloadString(node.SelectSingleNode("//a").GetAttributeValue("href", ""));
last_change = true;
}
}
}
Console.WriteLine("Complete");
var result = new List <HitomiArticle>();
for (int i = 0, j = 0; i < gburls.Count; i++)
{
if (htmls[i] == null)
{
continue;
}
var aa = HitomiParser.ParseGalleryBlock(htmls[i]);
if (htmls2[j] != null)
{
var ab = HitomiParser.ParseGallery(htmls2[j]);
aa.Groups = ab.Groups;
aa.Characters = ab.Characters;
}
result.Add(aa);
j++;
}
Console.Write("Save to hiddendata.json... ");
HitomiData.Instance.SaveWithNewData(result);
Console.WriteLine("Complete");
//#if true
// Console.Write("Save to index-metadata.json... ");
// HitomiIndex.MakeIndex();
// Console.WriteLine("Complete");
//#endif
}
//#if false
// Sync EH
{
var result = new List <EHentaiResultArticle>();
for (int i = 0; i < 9999999; i++)
{
try
{
//var task = NetTask.MakeDefault($"https://exhentai.org/?page={i}&f_doujinshi=on&f_manga=on&f_artistcg=on&f_gamecg=on&&f_cats=0&f_sname=on&f_stags=on&f_sh=on&advsearch=1&f_srdd=2&f_sname=on&f_stags=on&f_sdesc=on&f_sh=on");
//task.Cookie = "igneous=30e0c0a66;ipb_member_id=2742770;ipb_pass_hash=6042be35e994fed920ee7dd11180b65f;sl=dm_2";
//var html = NetTools.DownloadString(task);
var url = $"https://exhentai.org/?page={i}&f_doujinshi=on&f_manga=on&f_artistcg=on&f_gamecg=on&&f_cats=0&f_sname=on&f_stags=on&f_sh=on&advsearch=1&f_srdd=2&f_sname=on&f_stags=on&f_sdesc=on&f_sh=on";
var wc = new WebClient();
wc.Encoding = Encoding.UTF8;
wc.Headers.Add(HttpRequestHeader.Accept, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
wc.Headers.Add(HttpRequestHeader.UserAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36");
wc.Headers.Add(HttpRequestHeader.Cookie, "igneous=30e0c0a66;ipb_member_id=2742770;ipb_pass_hash=6042be35e994fed920ee7dd11180b65f;sl=dm_2");
var html = wc.DownloadString(url);
try
{
var exh = ExHentaiParser.ParseResultPageExtendedListView(html);
result.AddRange(exh);
if (exh.Count != 25)
{
Logs.Instance.PushWarning("[Miss] " + url);
}
if (i > 500 && exh.Min(x => x.URL.Split('/')[4].ToInt()) < latest)
{
break;
}
Logs.Instance.Push("Parse exh page - " + i);
}
catch (Exception e)
{
Logs.Instance.PushError("[Fail] " + url);
}
}
catch (Exception e)
{
Logs.Instance.PushError($"{i} {e.Message}");
}
Thread.Sleep(100);
if (i % 1000 == 999)
{
Thread.Sleep(60000);
}
}
var xxx = JsonConvert.DeserializeObject <List <EHentaiResultArticle> >(File.ReadAllText("ex-hentai-archive.json"));
File.Move("ex-hentai-archive.json", $"ex-hentai-archive-{DateTime.Now.Ticks}.json");
var exists = new HashSet <int>();
xxx.ForEach(x => exists.Add(x.URL.Split('/')[4].ToInt()));
foreach (var z in result)
{
var nn = z.URL.Split('/')[4].ToInt();
if (!exists.Contains(nn))
{
xxx.Add(z);
}
}
JsonSerializer serializer = new JsonSerializer();
serializer.Converters.Add(new JavaScriptDateTimeConverter());
serializer.NullValueHandling = NullValueHandling.Ignore;
Logs.Instance.Push("Write file: ex-hentai-archive.json");
using (StreamWriter sw = new StreamWriter("ex-hentai-archive.json"))
using (JsonWriter writer = new JsonTextWriter(sw))
{
serializer.Serialize(writer, xxx);
}
}
//#endif
//#endif
//#if false
// Make DataBase
{
HitomiData.Instance.metadata_collection.Clear();
HitomiData.Instance.Load();
var xxx = JsonConvert.DeserializeObject <List <EHentaiResultArticle> >(File.ReadAllText("ex-hentai-archive.json"));
Console.Write("Make database... ");
var dict = new Dictionary <string, int>();
for (int i = 0; i < xxx.Count; i++)
{
if (!dict.ContainsKey(xxx[i].URL.Split('/')[4]))
{
dict.Add(xxx[i].URL.Split('/')[4], i);
}
}
{
var db = new SQLiteConnection("hitomidata.db");
var info = db.GetTableInfo(typeof(HitomiColumnModel).Name);
if (!info.Any())
{
db.CreateTable <HitomiColumnModel>();
}
db.InsertAll(HitomiData.Instance.metadata_collection.Select(md =>
{
var dd = new HitomiColumnModel
{
Id = md.ID,
Artists = (md.Artists != null && md.Artists.Length > 0 && md.Artists[0] != "" ? "|" + string.Join("|", md.Artists) + "|" : "N/A|"),
Characters = (md.Characters != null && md.Characters.Length > 0 && md.Characters[0] != "" ? "|" + string.Join("|", md.Characters) + "|" : null),
Groups = (md.Groups != null && md.Groups.Length > 0 && md.Groups[0] != "" ? "|" + string.Join("|", md.Groups) + "|" : null),
Series = (md.Parodies != null && md.Parodies.Length > 0 && md.Parodies[0] != "" ? "|" + string.Join("|", md.Parodies) + "|" : null),
Title = md.Name,
Tags = (md.Tags != null && md.Tags.Length > 0 && md.Tags[0] != "" ? "|" + string.Join("|", md.Tags) + "|" : null),
Type = md.Type,
Language = md.Language,
};
if (dict.ContainsKey(md.ID.ToString()))
{
var ii = xxx[dict[md.ID.ToString()]];
dd.Uploader = ii.Uploader;
dd.Published = DateTime.Parse(ii.Published);
dd.EHash = ii.URL.Split('/')[5];
dd.Files = ii.Files.Split(' ')[0].ToInt();
if (ii.Title.StartsWith("("))
{
dd.Class = ii.Title.Split("(")[1].Split(")")[0];
}
}
return(dd);
}));
db.Close();
}
Console.WriteLine("Complete-All");
{
var db = new SQLiteConnection("hitomidata-korean.db");
var info = db.GetTableInfo(typeof(HitomiColumnModel).Name);
if (!info.Any())
{
db.CreateTable <HitomiColumnModel>();
}
db.InsertAll(HitomiData.Instance.metadata_collection.Where(md => md.Language == null || md.Language == "" || md.Language == "korean").Select(md =>
{
var dd = new HitomiColumnModel
{
Id = md.ID,
Artists = (md.Artists != null && md.Artists.Length > 0 && md.Artists[0] != "" ? "|" + string.Join("|", md.Artists) + "|" : "N/A|"),
Characters = (md.Characters != null && md.Characters.Length > 0 && md.Characters[0] != "" ? "|" + string.Join("|", md.Characters) + "|" : null),
Groups = (md.Groups != null && md.Groups.Length > 0 && md.Groups[0] != "" ? "|" + string.Join("|", md.Groups) + "|" : null),
Series = (md.Parodies != null && md.Parodies.Length > 0 && md.Parodies[0] != "" ? "|" + string.Join("|", md.Parodies) + "|" : null),
Title = md.Name,
Tags = (md.Tags != null && md.Tags.Length > 0 && md.Tags[0] != "" ? "|" + string.Join("|", md.Tags) + "|" : null),
Type = md.Type,
Language = md.Language,
};
if (dict.ContainsKey(md.ID.ToString()))
{
var ii = xxx[dict[md.ID.ToString()]];
dd.Uploader = ii.Uploader;
dd.Published = DateTime.Parse(ii.Published);
dd.EHash = ii.URL.Split('/')[5];
dd.Files = ii.Files.Split(' ')[0].ToInt();
if (ii.Title.StartsWith("("))
{
dd.Class = ii.Title.Split("(")[1].Split(")")[0];
}
}
return(dd);
}));
db.Close();
}
Console.WriteLine("Complete-Korean");
{
var db = new SQLiteConnection("hitomidata-japanese.db");
var info = db.GetTableInfo(typeof(HitomiColumnModel).Name);
if (!info.Any())
{
db.CreateTable <HitomiColumnModel>();
}
db.InsertAll(HitomiData.Instance.metadata_collection.Where(md => md.Language == null || md.Language == "" || md.Language == "japanese").Select(md =>
{
var dd = new HitomiColumnModel
{
Id = md.ID,
Artists = (md.Artists != null && md.Artists.Length > 0 && md.Artists[0] != "" ? "|" + string.Join("|", md.Artists) + "|" : "N/A|"),
Characters = (md.Characters != null && md.Characters.Length > 0 && md.Characters[0] != "" ? "|" + string.Join("|", md.Characters) + "|" : null),
Groups = (md.Groups != null && md.Groups.Length > 0 && md.Groups[0] != "" ? "|" + string.Join("|", md.Groups) + "|" : null),
Series = (md.Parodies != null && md.Parodies.Length > 0 && md.Parodies[0] != "" ? "|" + string.Join("|", md.Parodies) + "|" : null),
Title = md.Name,
Tags = (md.Tags != null && md.Tags.Length > 0 && md.Tags[0] != "" ? "|" + string.Join("|", md.Tags) + "|" : null),
Type = md.Type,
Language = md.Language,
};
if (dict.ContainsKey(md.ID.ToString()))
{
var ii = xxx[dict[md.ID.ToString()]];
dd.Uploader = ii.Uploader;
dd.Published = DateTime.Parse(ii.Published);
dd.EHash = ii.URL.Split('/')[5];
dd.Files = ii.Files.Split(' ')[0].ToInt();
if (ii.Title.StartsWith("("))
{
dd.Class = ii.Title.Split("(")[1].Split(")")[0];
}
}
return(dd);
}));
db.Close();
}
Console.WriteLine("Complete-Japanese");
{
var db = new SQLiteConnection("hitomidata-english.db");
var info = db.GetTableInfo(typeof(HitomiColumnModel).Name);
if (!info.Any())
{
db.CreateTable <HitomiColumnModel>();
}
db.InsertAll(HitomiData.Instance.metadata_collection.Where(md => md.Language == null || md.Language == "" || md.Language == "english").Select(md =>
{
var dd = new HitomiColumnModel
{
Id = md.ID,
Artists = (md.Artists != null && md.Artists.Length > 0 && md.Artists[0] != "" ? "|" + string.Join("|", md.Artists) + "|" : "N/A|"),
Characters = (md.Characters != null && md.Characters.Length > 0 && md.Characters[0] != "" ? "|" + string.Join("|", md.Characters) + "|" : null),
Groups = (md.Groups != null && md.Groups.Length > 0 && md.Groups[0] != "" ? "|" + string.Join("|", md.Groups) + "|" : null),
Series = (md.Parodies != null && md.Parodies.Length > 0 && md.Parodies[0] != "" ? "|" + string.Join("|", md.Parodies) + "|" : null),
Title = md.Name,
Tags = (md.Tags != null && md.Tags.Length > 0 && md.Tags[0] != "" ? "|" + string.Join("|", md.Tags) + "|" : null),
Type = md.Type,
Language = md.Language,
};
if (dict.ContainsKey(md.ID.ToString()))
{
var ii = xxx[dict[md.ID.ToString()]];
dd.Uploader = ii.Uploader;
dd.Published = DateTime.Parse(ii.Published);
dd.EHash = ii.URL.Split('/')[5];
dd.Files = ii.Files.Split(' ')[0].ToInt();
if (ii.Title.StartsWith("("))
{
dd.Class = ii.Title.Split("(")[1].Split(")")[0];
}
}
return(dd);
}));
db.Close();
}
Console.WriteLine("Complete-English");
{
var db = new SQLiteConnection("hitomidata-chinese.db");
var info = db.GetTableInfo(typeof(HitomiColumnModel).Name);
if (!info.Any())
{
db.CreateTable <HitomiColumnModel>();
}
db.InsertAll(HitomiData.Instance.metadata_collection.Where(md => md.Language == null || md.Language == "" || md.Language == "chinese").Select(md =>
{
var dd = new HitomiColumnModel
{
Id = md.ID,
Artists = (md.Artists != null && md.Artists.Length > 0 && md.Artists[0] != "" ? "|" + string.Join("|", md.Artists) + "|" : "N/A|"),
Characters = (md.Characters != null && md.Characters.Length > 0 && md.Characters[0] != "" ? "|" + string.Join("|", md.Characters) + "|" : null),
Groups = (md.Groups != null && md.Groups.Length > 0 && md.Groups[0] != "" ? "|" + string.Join("|", md.Groups) + "|" : null),
Series = (md.Parodies != null && md.Parodies.Length > 0 && md.Parodies[0] != "" ? "|" + string.Join("|", md.Parodies) + "|" : null),
Title = md.Name,
Tags = (md.Tags != null && md.Tags.Length > 0 && md.Tags[0] != "" ? "|" + string.Join("|", md.Tags) + "|" : null),
Type = md.Type,
Language = md.Language,
};
if (dict.ContainsKey(md.ID.ToString()))
{
var ii = xxx[dict[md.ID.ToString()]];
dd.Uploader = ii.Uploader;
dd.Published = DateTime.Parse(ii.Published);
dd.EHash = ii.URL.Split('/')[5];
dd.Files = ii.Files.Split(' ')[0].ToInt();
if (ii.Title.StartsWith("("))
{
dd.Class = ii.Title.Split("(")[1].Split(")")[0];
}
}
return(dd);
}));
db.Close();
}
Console.WriteLine("Complete-Chinese");
}
//#endif
}
