public DataBaseCreator()
{
// Load or Reload Raw datas
if (HitomiData.Instance.metadata_collection != null)
HitomiData.Instance.metadata_collection.Clear();
HitomiData.Instance.Load();
ehentaiArticles = JsonConvert.DeserializeObject<List<EHentaiResultArticle>>(File.ReadAllText("ex-hentai-archive.json"));
// List of all works on ehentai
ehIndex = new Dictionary<string, int>();
// Minimum datetime
mindd = ehentaiArticles.Min(x => DateTime.Parse(x.Published));
var xx1 = new List<double>();
var yy1 = new List<double>();
for (int i = 0; i < ehentaiArticles.Count; i++)
{
xx1.Add(int.Parse(ehentaiArticles[i].URL.Split('/')[4]));
yy1.Add((DateTime.Parse(ehentaiArticles[i].Published) - mindd).TotalMinutes);
if (!ehIndex.ContainsKey(ehentaiArticles[i].URL.Split('/')[4]))
ehIndex.Add(ehentaiArticles[i].URL.Split('/')[4], i);
}
// Estimate DateTime
datetimeEstimator = new PolynomialRegressionModel(Vector.Create(yy1.ToArray()), Vector.Create(xx1.ToArray()), 100);
datetimeEstimator.Fit();
}
static void ProcessCreateDateTimeEstimator(bool low_perf)
{
PolynomialRegressionModel datetimeEstimator = null;
DateTime mindd;
var xx1 = new List <double>();
var yy1 = new List <double>();
if (!low_perf)
{
var ehentaiArticles = JsonConvert.DeserializeObject <List <EHentaiResultArticle> >(File.ReadAllText("ex-hentai-archive.json"));
mindd = ehentaiArticles.Min(x => DateTime.Parse(x.Published));
for (int i = 0; i < ehentaiArticles.Count; i++)
{
xx1.Add(int.Parse(ehentaiArticles[i].URL.Split('/')[4]));
yy1.Add((DateTime.Parse(ehentaiArticles[i].Published) - mindd).TotalMinutes);
}
}
else
{
var db = new SQLiteConnection("data.db");
var count = db.ExecuteScalar <int>("SELECT COUNT(*) FROM HitomiColumnModel");
const int perLoop = 50000;
var dts = new List <DateTime>();
for (int i = 0; i < count; i += perLoop)
{
var query = db.Query <HitomiColumnModel>($"SELECT * FROM HitomiColumnModel ORDER BY Id LIMIT {perLoop} OFFSET {i}");
foreach (var article in query)
{
if (article.Published.HasValue)
{
xx1.Add(article.Id);
dts.Add(article.Published.Value);
}
}
}
mindd = dts.Min();
for (int i = 0; i < count; i++)
{
yy1.Add((dts[i] - mindd).TotalMinutes);
}
}
datetimeEstimator = new PolynomialRegressionModel(Vector.Create(yy1.ToArray()), Vector.Create(xx1.ToArray()), 100);
datetimeEstimator.Fit();
var poly = datetimeEstimator.GetRegressionPolynomial();
var x = poly.Parameters.ToList();
var y = mindd.Subtract(
new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds;
var jt = new JObject();
jt.Add("base", y);
jt.Add("coff", JToken.FromObject(x));
if (File.Exists("dt-coff.json"))
{
File.Delete("dt-coff.json");
}
File.WriteAllText("dt-coff.json", jt.ToString());
}