public ImportEvent CreateImportEvent(int import_id, ISource source)
{
ImportEvent import = new ImportEvent(import_id, source.id);
if (source.has_study_tables)
{
import.num_new_studies = GetScalarDBValue("to_ad_study_recs", 1);
import.num_edited_studies = GetScalarDBValue("to_ad_study_recs", 2);
import.num_unchanged_studies = GetScalarDBValue("to_ad_study_recs", 3);
import.num_deleted_studies = GetScalarDBValue("to_ad_study_recs", 4);
}
else
{
import.num_new_studies = 0;
import.num_edited_studies = 0;
import.num_unchanged_studies = 0;
import.num_deleted_studies = 0;
}
import.num_new_objects = GetScalarDBValue("to_ad_object_recs", 1);
import.num_edited_objects = GetScalarDBValue("to_ad_object_recs", 2);;
import.num_unchanged_objects = GetScalarDBValue("to_ad_object_recs", 3);;
import.num_deleted_objects = GetScalarDBValue("to_ad_object_recs", 4);;
return(import);
}
public int StoreImportEvent(ImportEvent import)
{
import.time_ended = DateTime.Now;
using (var conn = new NpgsqlConnection(connString))
{
return((int)conn.Insert <ImportEvent>(import));
}
}
public HistoryRecord(ImportEvent imp)
{
id = imp.id;
num_new_studies = imp.num_new_studies;
num_edited_studies = imp.num_edited_studies;
num_unchanged_studies = imp.num_unchanged_studies;
num_deleted_studies = imp.num_deleted_studies;
num_new_objects = imp.num_new_objects;
num_edited_objects = imp.num_edited_objects;
num_unchanged_objects = imp.num_unchanged_objects;
num_deleted_objects = imp.num_deleted_objects;
time_created = DateTime.Now;
}
private void ImportData(int source_id, bool rebuild_ad_tables, bool using_test_data)
{
// Obtain source details, augment with connection string for this database.
ISource source = _mon_repo.FetchSourceParameters(source_id);
Credentials creds = _mon_repo.Credentials;
source.db_conn = creds.GetConnectionString(source.database_name, using_test_data);
if (using_test_data)
{
// first need to copy sd data back from composite
// sd tables to the sd tables...
_test_repo.RetrieveSDData(source);
}
_logger_helper.Logheader("Setup");
_logger.Information("For source: " + source.id + ": " + source.database_name);
// Establish top level builder classes and
// set up sf monitor tables as foreign tables, temporarily.
ImportBuilder ib = new ImportBuilder(source, _mon_repo, _logger);
DataTransferrer transferrer = new DataTransferrer(source, _logger);
transferrer.EstablishForeignMonTables(creds);
_logger.Information("Foreign (mon) tables established in database");
// Recreate ad tables if necessary. If the second pass of a
// test loop will need to retrieve the ad data back from compad
if (rebuild_ad_tables)
{
ADBuilder adb = new ADBuilder(source, _mon_repo, _logger);
adb.BuildNewADTables();
}
else
{
if (using_test_data)
{
_test_repo.RetrieveADData(source);
}
}
// create and fill temporary tables to hold ids and edit statuses
// of new, edited, deleted tudies and data objects.
_logger_helper.Logheader("Start Import Process");
_logger_helper.Logheader("Create and fill diff tables");
ib.CreateImportTables();
bool count_deleted = false;
if (!using_test_data)
{
count_deleted = _mon_repo.CheckIfFullHarvest(source.id);
}
ib.FillImportTables(count_deleted);
_mon_repo.LogDiffs(source);
// Create import event log record and start
// the data transfer proper...
int import_id = _mon_repo.GetNextImportEventId();
ImportEvent import = ib.CreateImportEvent(import_id);
// Consider new studies, record dates, edited studies and / or objects,
// and any deleted studies / objects
_logger_helper.Logheader("Adding new data");
if (source.has_study_tables)
{
transferrer.AddNewStudies(import_id);
}
transferrer.AddNewDataObjects(import_id);
_logger_helper.Logheader("Updating dates of data");
transferrer.UpdateDatesOfData();
_logger_helper.Logheader("Editing existing data where necessary");
if (source.has_study_tables)
{
transferrer.UpdateEditedStudyData(import_id);
}
transferrer.UpdateEditedDataObjectData(import_id);
_logger_helper.Logheader("Deleting data no longer present in source");
if (source.has_study_tables)
{
transferrer.RemoveDeletedStudyData(import_id);
}
transferrer.RemoveDeletedDataObjectData(import_id);
// Update the 'date imported' record in the mon.source data tables
// Affects all records with status 1, 2 or 3 (non-test imports only)
if (!using_test_data)
{
if (source.has_study_tables)
{
transferrer.UpdateStudiesLastImportedDate(import_id);
}
else
{
// only do the objects table if there are no studies (e.g. PubMed)
transferrer.UpdateObjectsLastImportedDate(import_id);
}
}
// Ensure that the full hash records have been updated
// may not have been if change was only in attribute(s).
// Remove foreign tables
_logger_helper.Logheader("Tidy up and finish");
transferrer.UpdateFullStudyHashes();
transferrer.DropForeignMonTables();
_logger.Information("Foreign (mon) tables removed from database");
if (using_test_data)
{
// copy ad data from ad tables to the compad tables...
_test_repo.TransferADDataToComp(source);
}
else
{
// Only store import event for non-test imports.
_mon_repo.StoreImportEvent(import);
}
}