public void CreateJSONStudyData(bool also_do_files, bool create_table = true, int offset = 0)
{
JSONStudyDataLayer repo = new JSONStudyDataLayer(logging_repo);
if (create_table)
{
string sql_string = @"DROP TABLE IF EXISTS core.studies_json;
CREATE TABLE core.studies_json(
id INT NOT NULL PRIMARY KEY
, json JSON NULL
);
CREATE INDEX studies_json_id ON core.studies_json(id);";
db.ExecuteSQL(sql_string);
}
int min_id = repo.FetchMinId();
int max_id = repo.FetchMaxId();
LoopThroughStudyRecords(repo, min_id, max_id, also_do_files, offset);
}
public void LoopThroughStudyRecords(JSONStudyDataLayer repo, int min_id, int max_id, bool also_do_files, int offset)
{
JSONStudyProcessor processor = new JSONStudyProcessor(repo);
// Do 10,000 ids at a time
int batch = 10000;
//int batch = 100; // testing
string folder_path = "";
int k = 0;
for (int n = min_id; n <= max_id; n += batch)
{
if (also_do_files)
{
// Create folder for the next batch, obtaining the parent path from repo
string folder_name = "studies " + n.ToString() + " to " + (n + batch - 1).ToString();
folder_path = Path.Combine(repo.StudyJsonFolder, folder_name);
if (!Directory.Exists(folder_path))
{
Directory.CreateDirectory(folder_path);
}
else
{
// first clear files from folder
DirectoryInfo di = new DirectoryInfo(folder_path);
foreach (FileInfo file in di.EnumerateFiles())
{
file.Delete();
}
}
}
IEnumerable <int> id_numbers = repo.FetchIds(n, batch);
foreach (int id in id_numbers)
{
// Construct single study object, drawing data from various database tables
// and serialise to a formatted json string, then store json in the database.
JSONStudy st = processor.CreateStudyObject(id);
if (st != null)
{
var linear_json = JsonConvert.SerializeObject(st);
processor.StoreJSONStudyInDB(id, linear_json);
if (also_do_files)
{
var formatted_json = JsonConvert.SerializeObject(st, Formatting.Indented);
string file_name = "study " + id.ToString() + ".json";
string full_path = Path.Combine(folder_path, file_name);
File.WriteAllText(full_path, formatted_json);
}
}
k++;
if (k % 1000 == 0)
{
logging_repo.LogLine(k.ToString() + " records processed");
}
}
}
}
public JSONStudyProcessor(JSONStudyDataLayer _repo)
{
repo = _repo;
}