public void DownloadGrantExpendituresTableForAllFiscalYears()
        {
            Logger.Info($"Starting '{JobName}' DownloadGrantExpendituresTableForAllFiscalYears");
            ClearOutdatedSocrataDataMartRawJsonImportsTableEntries();

            // See how current the data is
            DateTime lastFinanceApiLoadDate = FinanceApiLastLoadUtil.GetLastLoadDate();

            // 2001 is the first biennium for which API returned data. This is WAY farther back than needed, but
            // harmless to overreach, since we only use data for which we have matching PI/PC.
            const int beginBienniumFiscalYear = 2001;

            const int bienniumStep = 2;

            // Go at least one biennium beyond the current one
            var endBienniumFiscalYear = CurrentBiennium.GetCurrentBienniumFiscalYearFromDatabase() + bienniumStep;

            // Step through all the desired Bienniums
            for (var bienniumFiscalYear = beginBienniumFiscalYear; bienniumFiscalYear <= endBienniumFiscalYear; bienniumFiscalYear += bienniumStep)
            {
                // Since only 2007 current blows up, we want to process all the other available years in the meantime.
                // This sends us and email reminding us something is wrong, and there is also good data about success in the tables itself.
                // -- SLG 6/27/2019 -- https://projects.sitkatech.com/projects/wa_dnr_forest_health_tracker/cards/1635
                try
                {
                    ImportExpendituresForGivenBienniumFiscalYear(bienniumFiscalYear, lastFinanceApiLoadDate);
                }
                catch (Exception e)
                {
                    Logger.Error($"Error importing Expenditures for Biennium Fiscal Year {bienniumFiscalYear}: {e.Message}");
                }
            }

            Logger.Info($"Ending '{JobName}' DownloadGrantExpendituresTableForAllFiscalYears");
        }
Exemplo n.º 2
0
        public void DownloadSocrataProjectCodeTable()
        {
            Logger.Info($"Starting '{JobName}' DownloadSocrataProjectCodeTable");
            ClearOutdatedSocrataDataMartRawJsonImportsTableEntries();

            // See how current the data is
            DateTime lastFinanceApiLoadDate = FinanceApiLastLoadUtil.GetLastLoadDate();

            var importInfo = LatestSuccessfulJsonImportInfoForBienniumAndImportTableType(SocrataDataMartRawJsonImportTableType.ProjectCode.SocrataDataMartRawJsonImportTableTypeID, null);

            // If we've already successfully imported the latest data available for this fiscal year, skip doing it again.
            if (importInfo != null && importInfo.FinanceApiLastLoadDate == lastFinanceApiLoadDate)
            {
                Logger.Info($"DownloadSocrataProjectCodeTable - ProjectCode table already current. Last import: {importInfo.JsonImportDate} - LastFinanceApiLoadDate: {lastFinanceApiLoadDate}");
                return;
            }

            // Pull JSON off the page into a (possibly huge) string
            var fullUrl = AddSocrataMaxLimitTagToUrl(ProjectCodeJsonSocrataBaseUrl);

            Logger.Info($"Retrieving ProjectCode JSON from URL: {fullUrl}");
            var projectCodeJson = DownloadSocrataUrlToString(fullUrl, SocrataDataMartRawJsonImportTableType.ProjectCode);

            Logger.Info($"ProjectCode JSON length: {projectCodeJson.Length}");
            // Push that string into a raw JSON string in the raw staging table
            var socrataDataMartRawJsonImportID = ShoveRawJsonStringIntoTable(SocrataDataMartRawJsonImportTableType.ProjectCode, lastFinanceApiLoadDate, null, projectCodeJson);

            Logger.Info($"New SocrataDataMartRawJsonImportID: {socrataDataMartRawJsonImportID}");

            try
            {
                // Use the JSON to refresh the Project Code table
                ProjectCodeImportJson(socrataDataMartRawJsonImportID);
            }
            catch (Exception e)
            {
                // Mark as failed in table
                MarkJsonImportStatus(socrataDataMartRawJsonImportID, JsonImportStatusType.ProcessingFailed);

                // add more debugging information to the exception and re-throw
                var exceptionWithMoreInfo = new ApplicationException($"ProjectCodeImportJson failed for SocrataDataMartRawJsonImportID {socrataDataMartRawJsonImportID}", e);
                throw exceptionWithMoreInfo;
            }
            // If we get this far, it's successfully imported, and we can mark it as such
            MarkJsonImportStatus(socrataDataMartRawJsonImportID, JsonImportStatusType.ProcessingSuceeded);

            Logger.Info($"Ending '{JobName}' DownloadSocrataProjectCodeTable");
        }