public void CreateJSONObjectData(bool also_do_files, bool create_table = true, int offset = 0) { JSONObjectDataLayer repo = new JSONObjectDataLayer(logging_repo); if (create_table) { string sql_string = @"DROP TABLE IF EXISTS core.objects_json; CREATE TABLE core.objects_json( id INT NOT NULL PRIMARY KEY , json JSON NULL ); CREATE INDEX objects_json_id ON core.objects_json(id);"; db.ExecuteSQL(sql_string); } int min_id = repo.FetchMinId(); int max_id = repo.FetchMaxId(); LoopThroughObjectRecords(repo, min_id, max_id, also_do_files, offset); }
public void LoopThroughObjectRecords(JSONObjectDataLayer repo, int min_id, int max_id, bool also_do_files, int offset) { JSONObjectProcessor processor = new JSONObjectProcessor(repo, logging_repo); // Do 10,000 ids at a time int batch = 10000; //int batch = 100; // testing string folder_path = ""; int k = offset; min_id += offset; for (int n = min_id; n <= max_id; n += batch) { //if (n > min_id + 200) break; // testing if (also_do_files) { // Create folder for the next batch, obtaining the parent path from repo string folder_name = "objects " + n.ToString() + " to " + (n + batch - 1).ToString(); folder_path = Path.Combine(repo.ObjectJsonFolder, folder_name); if (!Directory.Exists(folder_path)) { Directory.CreateDirectory(folder_path); } else { // first clear files from folder DirectoryInfo di = new DirectoryInfo(folder_path); foreach (FileInfo file in di.EnumerateFiles()) { file.Delete(); } } } IEnumerable <int> id_numbers = repo.FetchIds(n, batch); foreach (int id in id_numbers) { // Construct single study object, drawing data from various database tables // and serialise to a formatted json string, then store json in the database. JSONDataObject obj = processor.CreateObject(id); if (obj != null) { var linear_json = JsonConvert.SerializeObject(obj); processor.StoreJSONObjectInDB(id, linear_json); if (also_do_files) { var formatted_json = JsonConvert.SerializeObject(obj, Formatting.Indented); string file_name = "object " + id.ToString() + ".json"; string full_path = Path.Combine(folder_path, file_name); File.WriteAllText(full_path, formatted_json); } } k++; if (k % 1000 == 0) { logging_repo.LogLine(k.ToString() + " records processed"); } } } }
public JSONObjectProcessor(JSONObjectDataLayer _repo, LoggingDataLayer _logging_repo) { repo = _repo; logging_repo = _logging_repo; }