public DataObject(FullDataObject fob)
 {
     sd_oid                 = fob.sd_oid;
     sd_sid                 = fob.sd_sid;
     title                  = fob.title;
     version                = fob.version;
     display_title          = fob.display_title;
     doi                    = fob.doi;
     doi_status_id          = fob.doi_status_id;
     publication_year       = fob.publication_year;
     object_class_id        = fob.object_class_id;
     object_class           = fob.object_class;
     object_type_id         = fob.object_type_id;
     object_type            = fob.object_type;
     managing_org_id        = fob.managing_org_id;
     managing_org           = fob.managing_org;
     lang_code              = "en";
     access_type_id         = fob.access_type_id;
     access_type            = fob.access_type;
     access_details         = fob.access_details;
     access_details_url     = fob.access_details_url;
     url_last_checked       = fob.url_last_checked;
     eosc_category          = fob.eosc_category;
     add_study_contribs     = fob.add_study_contribs;
     add_study_topics       = fob.add_study_topics;
     datetime_of_data_fetch = fob.datetime_of_data_fetch;
 }
示例#2
0
        public int?LoopThroughFiles(int harvest_type_id, int harvest_id)
        {
            // Loop through the available records a chunk at a time (may be 1 for smaller record sources)
            // First get the total number of records in the system for this source
            // Set up the outer limit and get the relevant records for each pass

            int total_amount = _mon_repo.FetchFileRecordsCount(_source.id, _source.source_type, harvest_type_id);
            int chunk        = _source.harvest_chunk;
            int k            = 0;

            for (int m = 0; m < total_amount; m += chunk)
            {
                // if (k > 2000) break; // for testing...

                IEnumerable <ObjectFileRecord> file_list = _mon_repo
                                                           .FetchObjectFileRecordsByOffset(_source.id, m, chunk, harvest_type_id);

                int n = 0; string filePath = "";
                foreach (ObjectFileRecord rec in file_list)
                {
                    // if (k > 50) break; // for testing...

                    n++; k++;
                    filePath = rec.local_path;
                    if (File.Exists(filePath))
                    {
                        XmlDocument xdoc = new XmlDocument();
                        xdoc.Load(filePath);
                        FullDataObject b = _processor.ProcessData(xdoc, rec.last_downloaded);

                        // store the data in the database
                        _storage_repo.StoreFullObject(b, _source);

                        // update file record with last processed datetime
                        // (if not in test mode)
                        if (harvest_type_id != 3)
                        {
                            _mon_repo.UpdateFileRecLastHarvested(rec.id, _source.source_type, harvest_id);
                        }
                    }

                    if (k % chunk == 0)
                    {
                        _logger.Information("Records harvested: " + k.ToString());
                    }
                }
            }

            return(k);
        }
示例#3
0
        public void StoreFullObject(FullDataObject b, ISource source)
        {
            db_conn = source.db_conn;
            ObjectCopyHelpers och = new ObjectCopyHelpers();

            DataObject d = new DataObject(b);

            using (var conn = new NpgsqlConnection(db_conn))
            {
                conn.Insert <DataObject>(d);
            }

            if (b.object_instances.Count > 0)
            {
                using (var conn = new NpgsqlConnection(db_conn))
                {
                    conn.Open();
                    och.object_instances_helper.SaveAll(conn, b.object_instances);
                }
            }

            if (b.object_titles.Count > 0)
            {
                using (var conn = new NpgsqlConnection(db_conn))
                {
                    conn.Open();
                    och.object_titles_helper.SaveAll(conn, b.object_titles);
                }
            }

            // these are database dependent

            if (source.has_object_dates && b.object_dates.Count > 0)
            {
                using (var conn = new NpgsqlConnection(db_conn))
                {
                    conn.Open();
                    och.object_dates_helper.SaveAll(conn, b.object_dates);
                }
            }

            if (source.has_object_relationships && b.object_relationships.Count > 0)
            {
                using (var conn = new NpgsqlConnection(db_conn))
                {
                    conn.Open();
                    och.object_relationships_helper.SaveAll(conn, b.object_relationships);
                }
            }

            if (source.has_object_rights && b.object_rights?.Any() == true)
            {
                using (var conn = new NpgsqlConnection(db_conn))
                {
                    conn.Open();
                    och.object_rights_helper.SaveAll(conn, b.object_rights);
                }
            }

            if (source.has_object_pubmed_set)
            {
                if (b.object_contributors?.Any() == true)
                {
                    using (var conn = new NpgsqlConnection(db_conn))
                    {
                        conn.Open();
                        och.object_contributors_helper.SaveAll(conn, b.object_contributors);
                    }
                }

                if (b.object_topics?.Any() == true)
                {
                    using (var conn = new NpgsqlConnection(db_conn))
                    {
                        conn.Open();
                        och.object_topics_helper.SaveAll(conn, b.object_topics);
                    }
                }

                if (b.object_comments?.Any() == true)
                {
                    using (var conn = new NpgsqlConnection(db_conn))
                    {
                        conn.Open();
                        och.object_comments_helper.SaveAll(conn, b.object_comments);
                    }
                }

                if (b.object_descriptions?.Any() == true)
                {
                    using (var conn = new NpgsqlConnection(db_conn))
                    {
                        conn.Open();
                        och.object_descriptions_helper.SaveAll(conn, b.object_descriptions);
                    }
                }

                if (b.object_identifiers?.Any() == true)
                {
                    using (var conn = new NpgsqlConnection(db_conn))
                    {
                        conn.Open();
                        och.object_identifiers_helper.SaveAll(conn, b.object_identifiers);
                    }
                }

                if (b.object_db_ids?.Any() == true)
                {
                    using (var conn = new NpgsqlConnection(db_conn))
                    {
                        conn.Open();
                        och.object_db_links_helper.SaveAll(conn, b.object_db_ids);
                    }
                }

                if (b.object_pubtypes?.Any() == true)
                {
                    using (var conn = new NpgsqlConnection(db_conn))
                    {
                        conn.Open();
                        och.publication_types_helper.SaveAll(conn, b.object_pubtypes);
                    }
                }


                if (b.journal_details != null)
                {
                    using (var conn = new NpgsqlConnection(db_conn))
                    {
                        conn.Insert <JournalDetails>(b.journal_details);
                    }
                }
            }
        }