public DataObject(FullDataObject fob)
{
sd_oid = fob.sd_oid;
sd_sid = fob.sd_sid;
title = fob.title;
version = fob.version;
display_title = fob.display_title;
doi = fob.doi;
doi_status_id = fob.doi_status_id;
publication_year = fob.publication_year;
object_class_id = fob.object_class_id;
object_class = fob.object_class;
object_type_id = fob.object_type_id;
object_type = fob.object_type;
managing_org_id = fob.managing_org_id;
managing_org = fob.managing_org;
lang_code = "en";
access_type_id = fob.access_type_id;
access_type = fob.access_type;
access_details = fob.access_details;
access_details_url = fob.access_details_url;
url_last_checked = fob.url_last_checked;
eosc_category = fob.eosc_category;
add_study_contribs = fob.add_study_contribs;
add_study_topics = fob.add_study_topics;
datetime_of_data_fetch = fob.datetime_of_data_fetch;
}
public int?LoopThroughFiles(int harvest_type_id, int harvest_id)
{
// Loop through the available records a chunk at a time (may be 1 for smaller record sources)
// First get the total number of records in the system for this source
// Set up the outer limit and get the relevant records for each pass
int total_amount = _mon_repo.FetchFileRecordsCount(_source.id, _source.source_type, harvest_type_id);
int chunk = _source.harvest_chunk;
int k = 0;
for (int m = 0; m < total_amount; m += chunk)
{
// if (k > 2000) break; // for testing...
IEnumerable <ObjectFileRecord> file_list = _mon_repo
.FetchObjectFileRecordsByOffset(_source.id, m, chunk, harvest_type_id);
int n = 0; string filePath = "";
foreach (ObjectFileRecord rec in file_list)
{
// if (k > 50) break; // for testing...
n++; k++;
filePath = rec.local_path;
if (File.Exists(filePath))
{
XmlDocument xdoc = new XmlDocument();
xdoc.Load(filePath);
FullDataObject b = _processor.ProcessData(xdoc, rec.last_downloaded);
// store the data in the database
_storage_repo.StoreFullObject(b, _source);
// update file record with last processed datetime
// (if not in test mode)
if (harvest_type_id != 3)
{
_mon_repo.UpdateFileRecLastHarvested(rec.id, _source.source_type, harvest_id);
}
}
if (k % chunk == 0)
{
_logger.Information("Records harvested: " + k.ToString());
}
}
}
return(k);
}
public void StoreFullObject(FullDataObject b, ISource source)
{
db_conn = source.db_conn;
ObjectCopyHelpers och = new ObjectCopyHelpers();
DataObject d = new DataObject(b);
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Insert <DataObject>(d);
}
if (b.object_instances.Count > 0)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_instances_helper.SaveAll(conn, b.object_instances);
}
}
if (b.object_titles.Count > 0)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_titles_helper.SaveAll(conn, b.object_titles);
}
}
// these are database dependent
if (source.has_object_dates && b.object_dates.Count > 0)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_dates_helper.SaveAll(conn, b.object_dates);
}
}
if (source.has_object_relationships && b.object_relationships.Count > 0)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_relationships_helper.SaveAll(conn, b.object_relationships);
}
}
if (source.has_object_rights && b.object_rights?.Any() == true)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_rights_helper.SaveAll(conn, b.object_rights);
}
}
if (source.has_object_pubmed_set)
{
if (b.object_contributors?.Any() == true)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_contributors_helper.SaveAll(conn, b.object_contributors);
}
}
if (b.object_topics?.Any() == true)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_topics_helper.SaveAll(conn, b.object_topics);
}
}
if (b.object_comments?.Any() == true)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_comments_helper.SaveAll(conn, b.object_comments);
}
}
if (b.object_descriptions?.Any() == true)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_descriptions_helper.SaveAll(conn, b.object_descriptions);
}
}
if (b.object_identifiers?.Any() == true)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_identifiers_helper.SaveAll(conn, b.object_identifiers);
}
}
if (b.object_db_ids?.Any() == true)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.object_db_links_helper.SaveAll(conn, b.object_db_ids);
}
}
if (b.object_pubtypes?.Any() == true)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Open();
och.publication_types_helper.SaveAll(conn, b.object_pubtypes);
}
}
if (b.journal_details != null)
{
using (var conn = new NpgsqlConnection(db_conn))
{
conn.Insert <JournalDetails>(b.journal_details);
}
}
}
}
