Пример #1
0
        public int?LoopThroughFiles(int harvest_type_id, int harvest_id)
        {
            // Loop through the available records a chunk at a time (may be 1 for smaller record sources)
            // First get the total number of records in the system for this source
            // Set up the outer limit and get the relevant records for each pass

            int total_amount = _mon_repo.FetchFileRecordsCount(_source.id, _source.source_type, harvest_type_id);
            int chunk        = _source.harvest_chunk;
            int k            = 0;

            for (int m = 0; m < total_amount; m += chunk)
            {
                // if (k > 2000) break; // for testing...

                IEnumerable <ObjectFileRecord> file_list = _mon_repo
                                                           .FetchObjectFileRecordsByOffset(_source.id, m, chunk, harvest_type_id);

                int n = 0; string filePath = "";
                foreach (ObjectFileRecord rec in file_list)
                {
                    // if (k > 50) break; // for testing...

                    n++; k++;
                    filePath = rec.local_path;
                    if (File.Exists(filePath))
                    {
                        XmlDocument xdoc = new XmlDocument();
                        xdoc.Load(filePath);
                        FullDataObject b = _processor.ProcessData(xdoc, rec.last_downloaded);

                        // store the data in the database
                        _storage_repo.StoreFullObject(b, _source);

                        // update file record with last processed datetime
                        // (if not in test mode)
                        if (harvest_type_id != 3)
                        {
                            _mon_repo.UpdateFileRecLastHarvested(rec.id, _source.source_type, harvest_id);
                        }
                    }

                    if (k % chunk == 0)
                    {
                        _logger.Information("Records harvested: " + k.ToString());
                    }
                }
            }

            return(k);
        }