private async Task <PublicSectorOrganisationLookUp> GetOnsOrganisations()
{
var maxHistoricFileAttempts = 5;
var attempt = 0;
var downloadSuccess = false;
while (attempt < maxHistoricFileAttempts)
{
var url = GetDownloadUrlForMonthYear(attempt);
_logger.Info($"Downloading ONS from {url}");
downloadSuccess = await _archiveDownloadService.DownloadFile(url, _workingFolder, _fileName);
if (downloadSuccess)
{
break;
}
attempt++;
}
if (!downloadSuccess)
{
const string errorMessage = "Failed to download ONS from current and previous month, potential URL format change";
_logger.Error(new Exception(errorMessage), errorMessage);
throw new Exception("Failed to download ONS from current and previous month, potential URL format change");
}
var excelFile = Path.Combine(_workingFolder, _fileName);
_logger.Info($"Reading ONS from {excelFile}");
var ol = _publicSectorOrganisationDatabaseUpdater.UpdateDatabase(excelFile);
return(ol);
}
public async Task <PublicSectorOrganisationLookUp> GetData()
{
var organisations = new List <PublicSectorOrganisation>();
foreach (var nhsLink in _configuration.NhsTrustsUrls)
{
var fileDownloadSuccessful = await _archiveDownloadService.DownloadFile(nhsLink, Path.GetTempPath(), "nhsfile.zip");
if (fileDownloadSuccessful)
{
var nhsZipFolderPath = Path.Combine(Path.GetTempPath(), "NhsExtract");
_archiveDownloadService.UnzipFile(Path.Combine(Path.GetTempPath(), "nhsFile.zip"), nhsZipFolderPath);
var fileName = _fileSystemRepository.GetDataFile(nhsZipFolderPath);
if (!string.IsNullOrEmpty(fileName))
{
organisations.AddRange(_nhsCsvReaderHelper.ReadNhsFile(fileName));
}
}
}
return(new PublicSectorOrganisationLookUp {
Organisations = organisations
});
}
public async Task RunUpdate()
{
_logger.Info("Executing CharityImporter");
//Default to June 2017
var importMonth = 06;
var importYear = 2017;
var lastImport = await _charityRepository.GetLastCharityDataImport();
if (lastImport != null)
{
//Target subsequent month's file
importMonth = lastImport.Month + 1;
importYear = lastImport.Year;
if (importMonth > 12)
{
importMonth = 1;
importYear = importYear + 1;
}
}
await _charityRepository.TruncateLoadTables();
var url = GetExtractUrlForMonthYear(importMonth, importYear);
var filename = GetFilenameForMonthYear(importMonth, importYear);
if (!await _archiveDownloadService.DownloadFile(url, _workingFolder, filename))
{
_logger.Error(new Exception($"Failed to download data from {url}"), $"Failed to download data from {url}");
return;
}
var zipFile = Path.Combine(_workingFolder, filename);
var extractPath = Path.Combine(_workingFolder, Path.GetFileNameWithoutExtension(filename));
_archiveDownloadService.UnzipFile(zipFile, extractPath);
_bcpService.ExecuteBcp(new BcpRequest
{
ServerName = _configuration.CharityBcpServerName,
UseTrustedConnection = _configuration.CharityBcpTrustedConnection,
Username = _configuration.CharityBcpUsername,
Password = _configuration.CharityBcpPassword,
TargetDb = _configuration.CharityBcpTargetDb,
TargetSchema = _configuration.CharityBcpTargetSchema,
RowTerminator = _configuration.CharityBcpRowTerminator,
FieldTerminator = _configuration.CharityBcpFieldTerminator,
SourceDirectory = _workingFolder + Path.GetFileNameWithoutExtension(filename)
});
//transfer data into data tables
_logger.Info("Transferring data from load tables");
await _charityRepository.ImportDataFromLoadTables();
//record import in db
_logger.Info("Recording successful import in database");
await _charityRepository.CreateCharityDataImport(importMonth, importYear);
}
private async Task <PublicSectorOrganisationLookUp> GetOnsOrganisations()
{
var url = GetDownloadUrlForMonthYear(false);
_logger.Info($"Downloading ONS from {url}");
if (!await _archiveDownloadService.DownloadFile(url, _workingFolder, _fileName))
{
_logger.Info($"Downloading ONS from {url}");
url = GetDownloadUrlForMonthYear(true);
if (!await _archiveDownloadService.DownloadFile(url, _workingFolder, _fileName))
{
const string errorMessage = "Failed to download ONS from current and previous month, potential URL format change";
_logger.Error(new Exception(errorMessage), errorMessage);
throw new Exception("Failed to download ONS from current and previous month, potential URL format change");
}
}
var excelFile = Path.Combine(_workingFolder, _fileName);
_logger.Info($"Reading ONS from {excelFile}");
var ol = _publicSectorOrganisationDatabaseUpdater.UpdateDatabase(excelFile);
return(ol);
}
