Esempio n. 1
0
        public void CheckForUpdates(string orgName, bool useBulk)
        {
            string downloadsFolder = Path.Combine(Path.GetDirectoryName(AppDomain.CurrentDomain.BaseDirectory), DOWNLOADS);

            if (!Directory.Exists(downloadsFolder))
            {
                Directory.CreateDirectory(downloadsFolder);
            }

            int totalErrors;
            int totalProcessed = totalErrors = 0;

            log.InfoFormat("Check for updates at {0}", EXPORTER_CATALOG);
            WebClient wc = new WebClient();

            using (MemoryStream ms = new MemoryStream(wc.DownloadData(EXPORTER_CATALOG)))
            {
                ms.Seek(0, SeekOrigin.Begin);
                HtmlDocument doc = new HtmlDocument();
                doc.Load(ms);

                List <string> links = (from link in doc.DocumentNode.SelectNodes("//a[@href]")
                                       select link.Attributes["href"]
                                       into href
                                       where !string.IsNullOrWhiteSpace(href.Value) && href.Value.Contains("_PRJ_X_")
                                       orderby href.Value
                                       select "http://exporter.nih.gov/" + href.Value).ToList();

                bool addToProcessed = false;
                foreach (string link in links)
                {
                    Uri    uri      = new Uri(link);
                    string fileName = Path.GetFileName(uri.LocalPath);
                    if (!FileProcessed(fileName))
                    {
                        log.InfoFormat("Downloading file {0}", fileName);
                        WebClient d         = new WebClient();
                        byte[]    compessed = d.DownloadData(uri);
//                        byte[] compessed = File.ReadAllBytes(@"F:\Downloads\RePORTER_PRJ_X_FY2012_016.zip");
                        log.InfoFormat("File '{0}' downloaded to {1} folder.", fileName, downloadsFolder);

                        string zipPath = Path.Combine(downloadsFolder, fileName);
                        File.WriteAllBytes(zipPath, compessed);
                        ZipFile file = new ZipFile(zipPath);
                        foreach (ZipEntry zipEntry in file)
                        {
                            string fName = zipEntry.FileName;
                            zipEntry.Extract(downloadsFolder, ExtractExistingFileAction.OverwriteSilently);
                            log.InfoFormat("Unpacked '{0}' to {1}.", fileName, fName);

                            try
                            {
                                if (useBulk)
                                {
                                    BulkImporter importer = new BulkImporter();
                                    importer.ImportData(Path.Combine(downloadsFolder, fName), orgName);
                                    log.InfoFormat("{0} Records imported. {1} Errors", importer.TotalRecords, importer.ErrorsCount);
                                    totalProcessed = totalProcessed + importer.TotalRecords;
                                    totalErrors    = totalErrors + importer.ErrorsCount;
                                    addToProcessed = true;
                                }
                                else
                                {
                                    GrantImporter gi = new GrantImporter();
                                    gi.ImportData(Path.Combine(downloadsFolder, fName), orgName, null);
                                    log.InfoFormat("{0} Records imported. {1} Errors", gi.TotalRecords, gi.ErrorsCount);
                                    totalProcessed = totalProcessed + gi.TotalRecords;
                                    totalErrors    = totalErrors + gi.ErrorsCount;
                                    addToProcessed = true;
                                }
                            }
                            catch (Exception ex)
                            {
                                log.Debug("Error running bulk insert.", ex);
                            }
                            finally
                            {
                                //add or not to add?
                                if (addToProcessed)
                                {
                                    AddFileToProcessed(fileName);
                                }
                            }
                        }
                    }
                }
                log.InfoFormat("End of import.");
                log.InfoFormat("Total: {0} Records imported with {1} Errors", totalProcessed, totalErrors);
            }
        }
Esempio n. 2
0
        private static void ExecTasks(Options options)
        {
            string fileName = null;

            if (!options.CheckForUpdates && !options.Validate && !options.CheckForUpdatesNoBulk)
            {
                if (options.FileName == null || options.FileName.Count == 0)
                {
                    Options.ShowUsage();
                    return;
                }

                fileName = options.FileName[0];
                if (!File.Exists(fileName))
                {
                    Options.ShowUsage();
                    log.InfoFormat("File {0} does not exists.", fileName);
                    Environment.ExitCode = 1;
                    return;
                }
            }

            Stopwatch sw = new Stopwatch();

            sw.Start();
            if (options.UseBCP)
            {
                BCPImporter bi = new BCPImporter();
                bi.Import(fileName);
            }
            else if (options.UseBULK)
            {
                BulkImporter bi = new BulkImporter();
                bi.ImportData(fileName, options.OrgName, null);
                log.InfoFormat("{0} Records imported. {1} Errors", bi.TotalProcessed, bi.ErrorsCount);
            }
            else if (options.CheckForUpdates)
            {
                WebDownloader d = new WebDownloader();
                d.CheckForUpdates(options.OrgName, true);
            }
            else if (options.CheckForUpdatesNoBulk)
            {
                WebDownloader d = new WebDownloader();
                d.CheckForUpdates(options.OrgName, false);
            }
            else if (!options.Validate)
            {
                GrantImporter gi = new GrantImporter();
                gi.ImportData(fileName, options.OrgName, null);
                log.InfoFormat("{0} Records imported. {1} Errors", gi.TotalRecords, gi.ErrorsCount);
            }
            if (options.Validate)
            {
                GrantOnlineValidator validator = new GrantOnlineValidator();
                validator.ValidateGrants();
                log.InfoFormat("Total {0} Record(s) validated. {1} invalid grant(s)", validator.TotalProcessed, validator.InvalidGrants);
                if (validator.ErrorsCount > 0)
                {
                    log.InfoFormat("Total {0} Errors(s)", validator.ErrorsCount);
                }
            }

            sw.Stop();
            log.InfoFormat("Total time: {0:0.#} seconds.", sw.Elapsed.TotalSeconds);
            log.Info("Done.");
            Environment.ExitCode = 0;
        }