public int ShoveRawJsonStringIntoTable(SocrataDataMartRawJsonImportTableType socrataDataMartRawJsonImportTableType,
DateTime lastFinanceApiLoadDate,
int?optionalBienniumFiscalYear,
string rawJsonString)
{
Logger.Info($"Starting '{JobName}' ShoveRawJsonStringIntoTable");
SocrataDataMartRawJsonImport newRawJsonImport = new SocrataDataMartRawJsonImport(DateTime.Now, socrataDataMartRawJsonImportTableType, rawJsonString, JsonImportStatusType.NotYetProcessed);
newRawJsonImport.FinanceApiLastLoadDate = lastFinanceApiLoadDate;
newRawJsonImport.BienniumFiscalYear = optionalBienniumFiscalYear;
HttpRequestStorage.DatabaseEntities.SocrataDataMartRawJsonImports.Add(newRawJsonImport);
// We use the System Person if none is available, because that indicates we are running from an automated context (Hangfire)
if (HttpRequestStorage.PersonIsSet())
{
HttpRequestStorage.DatabaseEntities.SaveChanges();
}
else
{
var systemUser = HttpRequestStorage.DatabaseEntities.People.GetSystemUser();
HttpRequestStorage.DatabaseEntities.SaveChanges(systemUser);
}
// Normally we might return the object here, but this thing is potentially so huge we want to dump it just as soon as we no longer need it.
Logger.Info($"Ending '{JobName}' ShoveRawJsonStringIntoTable");
return(newRawJsonImport.SocrataDataMartRawJsonImportID);
}
/// <summary>
/// Download the contents of the given URL to a temp file
/// </summary>
/// <returns>Full path of the temp file created that contains the contents of the URL</returns>
public string DownloadSocrataUrlToString(Uri urlToDownload, SocrataDataMartRawJsonImportTableType socrataDataMartRawJsonImportTableType)
{
Logger.Info($"Starting '{JobName}' DownloadSocrataUrlToString");
try
{
using (var webClient = new WebClient())
{
// This isn't needed with public APIs, but may help to prevent throttling, and let's the other side know who we are in a polite way.
// See: http://xxxxx
webClient.Headers.Add("X-App-Token", MultiTenantHelpers.GetSocrataAppToken());
string socrataJsonData = webClient.DownloadString(urlToDownload);
Logger.Info($"Ending '{JobName}' DownloadSocrataUrlToString");
return(socrataJsonData);
}
}
catch (Exception exception)
{
string errorMessage = $"Error downloading Socrata type {socrataDataMartRawJsonImportTableType.SocrataDataMartRawJsonImportTableTypeName}, URL {urlToDownload}: {exception.Message}";
Logger.Error(errorMessage);
throw;
}
}
