public void CreateJSONObjectData(bool also_do_files, bool create_table = true, int offset = 0)
{
JSONObjectDataLayer repo = new JSONObjectDataLayer(logging_repo);
if (create_table)
{
string sql_string = @"DROP TABLE IF EXISTS core.objects_json;
CREATE TABLE core.objects_json(
id INT NOT NULL PRIMARY KEY
, json JSON NULL
);
CREATE INDEX objects_json_id ON core.objects_json(id);";
db.ExecuteSQL(sql_string);
}
int min_id = repo.FetchMinId();
int max_id = repo.FetchMaxId();
LoopThroughObjectRecords(repo, min_id, max_id, also_do_files, offset);
}
public void LoopThroughObjectRecords(JSONObjectDataLayer repo, int min_id, int max_id, bool also_do_files, int offset)
{
JSONObjectProcessor processor = new JSONObjectProcessor(repo, logging_repo);
// Do 10,000 ids at a time
int batch = 10000;
//int batch = 100; // testing
string folder_path = "";
int k = offset;
min_id += offset;
for (int n = min_id; n <= max_id; n += batch)
{
//if (n > min_id + 200) break; // testing
if (also_do_files)
{
// Create folder for the next batch, obtaining the parent path from repo
string folder_name = "objects " + n.ToString() + " to " + (n + batch - 1).ToString();
folder_path = Path.Combine(repo.ObjectJsonFolder, folder_name);
if (!Directory.Exists(folder_path))
{
Directory.CreateDirectory(folder_path);
}
else
{
// first clear files from folder
DirectoryInfo di = new DirectoryInfo(folder_path);
foreach (FileInfo file in di.EnumerateFiles())
{
file.Delete();
}
}
}
IEnumerable <int> id_numbers = repo.FetchIds(n, batch);
foreach (int id in id_numbers)
{
// Construct single study object, drawing data from various database tables
// and serialise to a formatted json string, then store json in the database.
JSONDataObject obj = processor.CreateObject(id);
if (obj != null)
{
var linear_json = JsonConvert.SerializeObject(obj);
processor.StoreJSONObjectInDB(id, linear_json);
if (also_do_files)
{
var formatted_json = JsonConvert.SerializeObject(obj, Formatting.Indented);
string file_name = "object " + id.ToString() + ".json";
string full_path = Path.Combine(folder_path, file_name);
File.WriteAllText(full_path, formatted_json);
}
}
k++;
if (k % 1000 == 0)
{
logging_repo.LogLine(k.ToString() + " records processed");
}
}
}
}
public JSONObjectProcessor(JSONObjectDataLayer _repo, LoggingDataLayer _logging_repo)
{
repo = _repo;
logging_repo = _logging_repo;
}