public DataBaseCreator()
        {
            // Load or Reload Raw datas
            if (HitomiData.Instance.metadata_collection != null)
                HitomiData.Instance.metadata_collection.Clear();
            HitomiData.Instance.Load();
            ehentaiArticles = JsonConvert.DeserializeObject<List<EHentaiResultArticle>>(File.ReadAllText("ex-hentai-archive.json"));

            // List of all works on ehentai
            ehIndex = new Dictionary<string, int>();

            // Minimum datetime
            mindd = ehentaiArticles.Min(x => DateTime.Parse(x.Published));

            var xx1 = new List<double>();
            var yy1 = new List<double>();

            for (int i = 0; i < ehentaiArticles.Count; i++)
            {
                xx1.Add(int.Parse(ehentaiArticles[i].URL.Split('/')[4]));
                yy1.Add((DateTime.Parse(ehentaiArticles[i].Published) - mindd).TotalMinutes);
                if (!ehIndex.ContainsKey(ehentaiArticles[i].URL.Split('/')[4]))
                    ehIndex.Add(ehentaiArticles[i].URL.Split('/')[4], i);
            }

            // Estimate DateTime
            datetimeEstimator = new PolynomialRegressionModel(Vector.Create(yy1.ToArray()), Vector.Create(xx1.ToArray()), 100);
            datetimeEstimator.Fit();
        }
Beispiel #2
0
        static void ProcessCreateDateTimeEstimator(bool low_perf)
        {
            PolynomialRegressionModel datetimeEstimator = null;
            DateTime mindd;

            var xx1 = new List <double>();
            var yy1 = new List <double>();

            if (!low_perf)
            {
                var ehentaiArticles = JsonConvert.DeserializeObject <List <EHentaiResultArticle> >(File.ReadAllText("ex-hentai-archive.json"));
                mindd = ehentaiArticles.Min(x => DateTime.Parse(x.Published));

                for (int i = 0; i < ehentaiArticles.Count; i++)
                {
                    xx1.Add(int.Parse(ehentaiArticles[i].URL.Split('/')[4]));
                    yy1.Add((DateTime.Parse(ehentaiArticles[i].Published) - mindd).TotalMinutes);
                }
            }
            else
            {
                var db    = new SQLiteConnection("data.db");
                var count = db.ExecuteScalar <int>("SELECT COUNT(*) FROM HitomiColumnModel");

                const int perLoop = 50000;

                var dts = new List <DateTime>();

                for (int i = 0; i < count; i += perLoop)
                {
                    var query = db.Query <HitomiColumnModel>($"SELECT * FROM HitomiColumnModel ORDER BY Id LIMIT {perLoop} OFFSET {i}");

                    foreach (var article in query)
                    {
                        if (article.Published.HasValue)
                        {
                            xx1.Add(article.Id);
                            dts.Add(article.Published.Value);
                        }
                    }
                }

                mindd = dts.Min();

                for (int i = 0; i < count; i++)
                {
                    yy1.Add((dts[i] - mindd).TotalMinutes);
                }
            }

            datetimeEstimator = new PolynomialRegressionModel(Vector.Create(yy1.ToArray()), Vector.Create(xx1.ToArray()), 100);
            datetimeEstimator.Fit();
            var poly = datetimeEstimator.GetRegressionPolynomial();
            var x    = poly.Parameters.ToList();

            var y = mindd.Subtract(
                new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds;

            var jt = new JObject();

            jt.Add("base", y);
            jt.Add("coff", JToken.FromObject(x));

            if (File.Exists("dt-coff.json"))
            {
                File.Delete("dt-coff.json");
            }
            File.WriteAllText("dt-coff.json", jt.ToString());
        }