public void LoopThroughStudyRecords(JSONStudyDataLayer repo, int min_id, int max_id, bool also_do_files, int offset)
{
JSONStudyProcessor processor = new JSONStudyProcessor(repo);
// Do 10,000 ids at a time
int batch = 10000;
//int batch = 100; // testing
string folder_path = "";
int k = 0;
for (int n = min_id; n <= max_id; n += batch)
{
if (also_do_files)
{
// Create folder for the next batch, obtaining the parent path from repo
string folder_name = "studies " + n.ToString() + " to " + (n + batch - 1).ToString();
folder_path = Path.Combine(repo.StudyJsonFolder, folder_name);
if (!Directory.Exists(folder_path))
{
Directory.CreateDirectory(folder_path);
}
else
{
// first clear files from folder
DirectoryInfo di = new DirectoryInfo(folder_path);
foreach (FileInfo file in di.EnumerateFiles())
{
file.Delete();
}
}
}
IEnumerable <int> id_numbers = repo.FetchIds(n, batch);
foreach (int id in id_numbers)
{
// Construct single study object, drawing data from various database tables
// and serialise to a formatted json string, then store json in the database.
JSONStudy st = processor.CreateStudyObject(id);
if (st != null)
{
var linear_json = JsonConvert.SerializeObject(st);
processor.StoreJSONStudyInDB(id, linear_json);
if (also_do_files)
{
var formatted_json = JsonConvert.SerializeObject(st, Formatting.Indented);
string file_name = "study " + id.ToString() + ".json";
string full_path = Path.Combine(folder_path, file_name);
File.WriteAllText(full_path, formatted_json);
}
}
k++;
if (k % 1000 == 0)
{
logging_repo.LogLine(k.ToString() + " records processed");
}
}
}
}
public JSONStudy CreateStudyObject(int id)
{
// Re-initialise these compound properties.
brief_description = null;
data_sharing_statement = null;
study_type = null;
study_status = null;
study_gender_elig = null;
min_age = null;
max_age = null;
study_identifiers = null;
study_titles = null;
study_topics = null;
study_features = null;
study_relationships = null;
linked_data_objects = null;
// Get the singleton study properties from DB
var s = repo.FetchDbStudy(id);
// Instantiate the top level lookup types
if (s.brief_description != null)
{
brief_description = new text_block(s.brief_description, s.bd_contains_html);
}
if (s.data_sharing_statement != null)
{
data_sharing_statement = new text_block(s.data_sharing_statement, s.dss_contains_html);
}
if (s.study_type_id != null)
{
study_type = new lookup(s.study_type_id, s.study_type);
}
if (s.study_status_id != null)
{
study_status = new lookup(s.study_status_id, s.study_status);
}
if (s.study_gender_elig_id != null)
{
study_gender_elig = new lookup(s.study_gender_elig_id, s.study_gender_elig);
}
if (s.min_age != null)
{
min_age = new age_param(s.min_age, s.min_age_units_id, s.min_age_units);
}
if (s.max_age != null)
{
max_age = new age_param(s.max_age, s.max_age_units_id, s.max_age_units);
}
// instantiate a (json) study object and
// fill it with study level details
JSONStudy jst = new JSONStudy(s.id, s.display_title, brief_description,
data_sharing_statement, study_type, study_status, s.study_enrolment,
study_gender_elig, min_age, max_age, s.provenance_string);
// add the study identifier details
var db_study_identifiers = new List <DBStudyIdentifier>(repo.FetchDbStudyIdentifiers(id));
if (db_study_identifiers.Count > 0)
{
study_identifiers = new List <study_identifier>();
foreach (DBStudyIdentifier t in db_study_identifiers)
{
study_identifiers.Add(new study_identifier(t.id, t.identifier_value,
new lookup(t.identifier_type_id, t.identifier_type),
new lookup(t.identifier_org_id, t.identifier_org),
t.identifier_date, t.identifier_link));
}
}
// add the study title details
var db_study_titles = new List <DBStudyTitle>(repo.FetchDbStudyTitles(id));
if (db_study_titles.Count > 0)
{
study_titles = new List <study_title>();
foreach (DBStudyTitle t in db_study_titles)
{
study_titles.Add(new study_title(t.id, new lookup(t.title_type_id, t.title_type), t.title_text,
t.lang_code, t.comments));
}
}
// add the study feature details
var db_study_features = new List <DBStudyFeature>(repo.FetchDbStudyFeatures(id));
if (db_study_features.Count > 0)
{
study_features = new List <study_feature>();
foreach (DBStudyFeature t in db_study_features)
{
study_features.Add(new study_feature(t.id, new lookup(t.feature_type_id, t.feature_type),
new lookup(t.feature_value_id, t.feature_value)));
}
}
// add the study topic details
var db_study_topics = new List <DBStudyTopic>(repo.FetchDbStudyTopics(id));
if (db_study_topics.Count > 0)
{
study_topics = new List <study_topic>();
foreach (DBStudyTopic t in db_study_topics)
{
study_topics.Add(new study_topic(t.id, new lookup(t.topic_type_id, t.topic_type),
t.mesh_coded, t.topic_code, t.topic_value, t.topic_qualcode,
t.topic_qualvalue, t.original_value));
}
}
// add the study relationships, if any
var db_study_relationships = new List <DBStudyRelationship>(repo.FetchDbStudyRelationships(id));
if (db_study_relationships.Count > 0)
{
study_relationships = new List <study_relationship>();
foreach (DBStudyRelationship t in db_study_relationships)
{
study_relationships.Add(new study_relationship(t.id,
new lookup(t.relationship_type_id, t.relationship_type),
t.target_study_id));
}
}
// add the related objects data
var db_study_object_links = new List <DBStudyObjectLink>(repo.FetchDbStudyObjectLinks(id));
if (db_study_object_links.Count > 0)
{
linked_data_objects = new List <int>();
foreach (DBStudyObjectLink t in db_study_object_links)
{
linked_data_objects.Add(t.object_id);
}
}
// return the resulting 'json ready' study
jst.study_identifiers = study_identifiers;
jst.study_titles = study_titles;
jst.study_features = study_features;
jst.study_topics = study_topics;
jst.study_relationships = study_relationships;
jst.linked_data_objects = linked_data_objects;
return(jst);
}
