public void DownloadGrantExpendituresTableForAllFiscalYears()
{
Logger.Info($"Starting '{JobName}' DownloadGrantExpendituresTableForAllFiscalYears");
ClearOutdatedSocrataDataMartRawJsonImportsTableEntries();
// See how current the data is
DateTime lastFinanceApiLoadDate = FinanceApiLastLoadUtil.GetLastLoadDate();
// 2001 is the first biennium for which API returned data. This is WAY farther back than needed, but
// harmless to overreach, since we only use data for which we have matching PI/PC.
const int beginBienniumFiscalYear = 2001;
const int bienniumStep = 2;
// Go at least one biennium beyond the current one
var endBienniumFiscalYear = CurrentBiennium.GetCurrentBienniumFiscalYearFromDatabase() + bienniumStep;
// Step through all the desired Bienniums
for (var bienniumFiscalYear = beginBienniumFiscalYear; bienniumFiscalYear <= endBienniumFiscalYear; bienniumFiscalYear += bienniumStep)
{
// Since only 2007 current blows up, we want to process all the other available years in the meantime.
// This sends us and email reminding us something is wrong, and there is also good data about success in the tables itself.
// -- SLG 6/27/2019 -- https://projects.sitkatech.com/projects/wa_dnr_forest_health_tracker/cards/1635
try
{
ImportExpendituresForGivenBienniumFiscalYear(bienniumFiscalYear, lastFinanceApiLoadDate);
}
catch (Exception e)
{
Logger.Error($"Error importing Expenditures for Biennium Fiscal Year {bienniumFiscalYear}: {e.Message}");
}
}
Logger.Info($"Ending '{JobName}' DownloadGrantExpendituresTableForAllFiscalYears");
}
public void DownloadSocrataProjectCodeTable()
{
Logger.Info($"Starting '{JobName}' DownloadSocrataProjectCodeTable");
ClearOutdatedSocrataDataMartRawJsonImportsTableEntries();
// See how current the data is
DateTime lastFinanceApiLoadDate = FinanceApiLastLoadUtil.GetLastLoadDate();
var importInfo = LatestSuccessfulJsonImportInfoForBienniumAndImportTableType(SocrataDataMartRawJsonImportTableType.ProjectCode.SocrataDataMartRawJsonImportTableTypeID, null);
// If we've already successfully imported the latest data available for this fiscal year, skip doing it again.
if (importInfo != null && importInfo.FinanceApiLastLoadDate == lastFinanceApiLoadDate)
{
Logger.Info($"DownloadSocrataProjectCodeTable - ProjectCode table already current. Last import: {importInfo.JsonImportDate} - LastFinanceApiLoadDate: {lastFinanceApiLoadDate}");
return;
}
// Pull JSON off the page into a (possibly huge) string
var fullUrl = AddSocrataMaxLimitTagToUrl(ProjectCodeJsonSocrataBaseUrl);
Logger.Info($"Retrieving ProjectCode JSON from URL: {fullUrl}");
var projectCodeJson = DownloadSocrataUrlToString(fullUrl, SocrataDataMartRawJsonImportTableType.ProjectCode);
Logger.Info($"ProjectCode JSON length: {projectCodeJson.Length}");
// Push that string into a raw JSON string in the raw staging table
var socrataDataMartRawJsonImportID = ShoveRawJsonStringIntoTable(SocrataDataMartRawJsonImportTableType.ProjectCode, lastFinanceApiLoadDate, null, projectCodeJson);
Logger.Info($"New SocrataDataMartRawJsonImportID: {socrataDataMartRawJsonImportID}");
try
{
// Use the JSON to refresh the Project Code table
ProjectCodeImportJson(socrataDataMartRawJsonImportID);
}
catch (Exception e)
{
// Mark as failed in table
MarkJsonImportStatus(socrataDataMartRawJsonImportID, JsonImportStatusType.ProcessingFailed);
// add more debugging information to the exception and re-throw
var exceptionWithMoreInfo = new ApplicationException($"ProjectCodeImportJson failed for SocrataDataMartRawJsonImportID {socrataDataMartRawJsonImportID}", e);
throw exceptionWithMoreInfo;
}
// If we get this far, it's successfully imported, and we can mark it as such
MarkJsonImportStatus(socrataDataMartRawJsonImportID, JsonImportStatusType.ProcessingSuceeded);
Logger.Info($"Ending '{JobName}' DownloadSocrataProjectCodeTable");
}