/// <summary>
/// Kills all currently running/pending tasks.
/// Releases all resources.
/// </summary>
public void Dispose()
{
lock (this)
{
if (!disposed)
{
startInitialThreads.Wait();
disposed = true;
ShutdownPool.Set();
//ts.Cancel();
// n.b. should not timeout, but you can't be too careful.
lock (RunningJobs)
{
Thread t = new Thread(() =>
{
// n.b. cannot WaitAll on STA thread
WaitHandle.WaitAll(RunningJobs.ToArray(), 30 * 1000);
});
t.SetApartmentState(ApartmentState.MTA);
t.Start();
t.Join();
}
ShutdownPool.Dispose();
startInitialThreads.Dispose();
}
}
}
private void CreateRunningJob(JobInfo jobInfo)
{
if (jobInfo == null)
{
return;
}
Tracer.Info("Starting workers for a new job. PipelineId = {0}, JobId = {1}, JobTypeId = {2}",
jobInfo.PipelineId, jobInfo.JobId, jobInfo.JobTypeId);
RunningJob runningJob = new RunningJob(jobInfo);
foreach (WorkRequest workRequest in jobInfo.WorkRequests)
{
workRequest.BootParameters = jobInfo.BootParameters;
workRequest.JobParameters = jobInfo.JobParameters;
WorkerCard workerCard = WorkersInventory.Instance.FindWorkerForRole(workRequest.RoleType);
if (null == workerCard)
{
Tracer.Info("This worker manager does not have workers suitable for the role {0}", workRequest.RoleType);
continue;
}
JobSection jobSection = new JobSection(workRequest, workerCard);
runningJob.Sections.Add(jobSection);
}
RunningJobs.Add(runningJob);
if (jobInfo.Command == Command.Run)
{
EmployWorkersForJob(runningJob);
}
}
private void EmergencyEvacuation()
{
Debug.Assert(null != RunningJobs);
if (RunningJobs.Count == 0)
{
return;
}
if (!declaredStateOfEmergency)
{
Tracer.Error("Director unavailable. Worker Manager starts emergency evacuation of workers.");
declaredStateOfEmergency = true;
}
List <RunningJob> updatedListOfRunningJobs = RunningJobs.Where(runningJob => StopWorkersForJob(runningJob)).ToList();
RunningJobs = updatedListOfRunningJobs;
if (RunningJobs.Count > 0)
{
return; // Wait for jobs to stop
}
Tracer.Warning("Director unavailable. All workers successfully evacuated from the factory.");
// All workers evacuated
}
/// <summary>
/// 监听触发
/// </summary>
/// <param name="connection">连接</param>
private void OnFire(HubConnection connection)
{
connection.On <JobContext, string>("Fire", (context, batchId) =>
{
var className = context?.Name;
try
{
bool shouldFire = false;
if (string.IsNullOrWhiteSpace(className) || !ClassNameMapTypes.ContainsKey(className) ||
(BypassRunning && RunningJobs.ContainsKey(className)))
{
return;
}
if ((context.FireTime - DateTime.Now).TotalSeconds <= 10)
{
shouldFire = true;
}
else
{
Debug.WriteLine($"触发任务超时: {className}");
}
if (shouldFire)
{
connection.SendAsync("FireCallback", batchId, context.Id, JobStatus.Running).Wait();
var jobType = ClassNameMapTypes[className];
RunningJobs.TryAdd(className, null);
Task.Factory.StartNew(() =>
{
bool success = false;
try
{
var jobObject = (IJobProcessor)Activator.CreateInstance(jobType);
success = jobObject.Process(context);
}
catch
{
// TODO: 输出日志
}
finally
{
connection.SendAsync("FireCallback", batchId, context.Id,
success ? JobStatus.Success : JobStatus.Fire).Wait();
}
}).ContinueWith((t) => { RunningJobs.TryRemove(className, out _); });
}
else
{
connection.SendAsync("FireCallback", batchId, context.Id, JobStatus.Bypass).Wait();
}
}
catch (Exception e)
{
Debug.WriteLine($"触发任务 {className} 失败: {e}");
}
});
}
public async Task UpdateRegistrationsAsync(ILogger log, string filePath)
{
if (RunningJobs.Contains(nameof(UpdateRegistrations)))
{
log.LogDebug($"'{nameof(UpdateRegistrations)}' is already running.");
return;
}
RunningJobs.Add(nameof(UpdateRegistrations));
try
{
var userOrgs = _SharedBusinessLogic.DataRepository.GetAll <UserOrganisation>().Where(uo =>
uo.User.Status == UserStatuses.Active && uo.PINConfirmedDate != null)
.OrderBy(uo => uo.Organisation.OrganisationName)
.Include(uo => uo.Organisation.LatestScope)
.Include(uo => uo.User)
.ToList();
var records = userOrgs.Select(
uo => new
{
uo.Organisation.OrganisationId,
uo.Organisation.DUNSNumber,
uo.Organisation.EmployerReference,
uo.Organisation.OrganisationName,
CompanyNo = uo.Organisation.CompanyNumber,
Sector = uo.Organisation.SectorType,
LatestReturn = uo.Organisation?.LatestReturn?.StatusDate,
uo.Method,
uo.Organisation.LatestScope?.ScopeStatus,
ScopeDate = uo.Organisation.LatestScope?.ScopeStatusDate,
uo.User.Fullname,
uo.User.JobTitle,
uo.User.EmailAddress,
uo.User.ContactFirstName,
uo.User.ContactLastName,
uo.User.ContactJobTitle,
uo.User.ContactEmailAddress,
uo.User.ContactPhoneNumber,
uo.User.ContactOrganisation,
uo.PINSentDate,
uo.PINConfirmedDate,
uo.Created,
Address = uo.Address?.GetAddressString()
})
.ToList();
await Extensions.SaveCSVAsync(_SharedBusinessLogic.FileRepository, records, filePath);
}
finally
{
RunningJobs.Remove(nameof(UpdateRegistrations));
}
}
private async Task UpdateSearchAsync <T>(ILogger log,
ISearchRepository <T> searchRepositoryToUpdate,
string indexNameToUpdate,
string userEmail = null,
bool force = false)
{
if (RunningJobs.Contains(nameof(UpdateSearchAsync)))
{
log.LogInformation("The set of running jobs already contains 'UpdateSearch'");
return;
}
try
{
await searchRepositoryToUpdate.CreateIndexIfNotExistsAsync(indexNameToUpdate);
if (typeof(T) == typeof(EmployerSearchModel))
{
await AddDataToIndexAsync(log);
}
else if (typeof(T) == typeof(SicCodeSearchModel))
{
await AddDataToSicCodesIndexAsync(log);
}
else
{
throw new ArgumentException($"Type {typeof(T)} is not a valid type.");
}
if (force && !string.IsNullOrWhiteSpace(userEmail))
{
try
{
await _Messenger.SendMessageAsync(
"UpdateSearchIndexes complete",
userEmail,
"The update of the search indexes completed successfully.");
}
catch (Exception ex)
{
log.LogError(ex, "UpdateSearch: An error occurred trying to send an email");
}
}
}
finally
{
RunningJobs.Remove(nameof(UpdateSearchAsync));
}
}
public async Task UpdateOrphanOrganisationsAsync(string filePath, ILogger log)
{
var funcName = nameof(UpdateOrphanOrganisationsAsync);
// Ensure the UpdateUnregisteredOrganisations web job is not already running
if (RunningJobs.Contains(funcName))
{
log.LogDebug($"Skipped {funcName} because already running.");
return;
}
try
{
// Flag the UpdateUnregisteredOrganisations web job as running
RunningJobs.Add(funcName);
// Cache the latest unregistered organisations
var unregisteredOrganisations = await GetOrphanOrganisationsAsync();
var year = _snapshotDateHelper.GetSnapshotDate(SectorTypes.Private).Year;
// Write yearly records to csv files
await WriteRecordsForYearAsync(
filePath,
year,
async() =>
{
foreach (var model in unregisteredOrganisations)
{
// get organisation scope and submission per year
var returnByYear =
await _SubmissionBusinessLogic.GetLatestSubmissionBySnapshotYearAsync(
model.OrganisationId, year);
var scopeByYear =
await _ScopeBusinessLogic.GetLatestScopeBySnapshotYearAsync(model.OrganisationId, year);
// update file model with year data
model.HasSubmitted = returnByYear == null ? "False" : "True";
model.ScopeStatus = scopeByYear?.ScopeStatus;
}
return(unregisteredOrganisations);
});
}
finally
{
RunningJobs.Remove(funcName);
}
}
public void RunningJobsFlush()
{
RunningJobs runs = new RunningJobs(persistencPath);
if (File.Exists(runs.PersistencePath))
{
File.Delete(runs.PersistencePath);
}
runs.Add(new JobRun(1, new TestIdJob()));
runs.Add(new JobRun(2, new TestIdJob()));
runs.Flush();
Assert.IsTrue(File.Exists(runs.PersistencePath));
}
public void RunningJobsLoad()
{
RunningJobs runs = new RunningJobs(persistencPath);
if (File.Exists(runs.PersistencePath))
{
File.Delete(runs.PersistencePath);
}
runs.Add(new JobRun(1, new TestIdJob()));
runs.Add(new JobRun(2, new TestIdJob()));
runs.Flush();
runs = new RunningJobs(persistencPath);
Assert.AreEqual(2, runs.Count);
}
/// <summary>
/// Remove jobs when they succeed/fail
/// </summary>
private void ProcessRunningJob(Job job)
{
if (!RunningJobs.TryGetValue(job, out var scrapeResult))
{
return;
}
lock (lockObject)
{
if (scrapeResult.IsCompleted)
{
var result = RunningJobs[job].Result;
if (result.Error != null)
{
job.Status = JobStatus.Failed;
job.Result = new Dictionary <string, IEnumerable <string> >
{
{ "Error", new List <string> {
scrapeResult.Exception.GetFullExceptionMessage()
} }
};
}
else
{
job.Status = JobStatus.Completed;
job.Result = result.Scrape;
}
JobRepository.Save(job);
RunningJobs.TryRemove(job, out var removedJob);
}
else if (scrapeResult.IsFaulted)
{
job.Status = JobStatus.Failed;
job.Result = new Dictionary <string, IEnumerable <string> >
{
{ "Error", new List <string> {
scrapeResult.Exception.GetFullExceptionMessage()
} }
};
JobRepository.Save(job);
RunningJobs.TryRemove(job, out var removedJob);
}
}
}
public void ShouldNotScheduleJobIfJobIsRunning()
{
// Arrange
RunningJobs runningJobs = new RunningJobs();
var scheduleManagerMock = new Mock <IScheduleJobManager>();
var siteSocialAccount = new SiteSocialAccount {
SiteId = 10000, FacebookPageId = "123"
};
Action <TriggerBuilder> configuerTriggerAction = t => { };
// Act
runningJobs.Schedule <TestJob>(scheduleManagerMock.Object, siteSocialAccount, configuerTriggerAction);
runningJobs.Schedule <TestJob>(scheduleManagerMock.Object, siteSocialAccount, configuerTriggerAction);
// Assert
scheduleManagerMock.Verify(mock => mock.ScheduleAsync <TestJob, SiteSocialAccount>(It.IsAny <Action <JobBuilder> >(), configuerTriggerAction, siteSocialAccount), Times.Once);
}
protected override void HasBeenActivated()
{
base.HasBeenActivated();
RunningJobs = crypCloudCore.GetJoblist();
RunningJobs.CollectionChanged += RunningJobs_CollectionChanged;
RaisePropertyChanged("RunningJobs");
if (RunningJobs.Count > 0)
{
selectedJob = RunningJobs.First();
RaisePropertyChanged("SelectedJob");
}
Contacts = crypCloudCore.GetContacts();
RaisePropertyChanged("Contacts");
Username = CrypCloudCore.Instance.GetUsername();
RaisePropertyChanged("Username");
RaisePropertyChanged("UserCanCreateJob");
}
public void ShouldRemoveRunningJob()
{
// Arrange
RunningJobs runningJobs = new RunningJobs();
var scheduleManagerMock = new Mock <IScheduleJobManager>();
var siteSocialAccount = new SiteSocialAccount {
SiteId = 10000, FacebookPageId = "123"
};
Action <TriggerBuilder> configuerTriggerAction = t => { };
// Act
runningJobs.Schedule <TestJob>(scheduleManagerMock.Object, siteSocialAccount, configuerTriggerAction);
runningJobs.Remove <TestJob>(siteSocialAccount.SiteId, siteSocialAccount.FacebookPageId);
var runningJob = runningJobs.Get <TestJob>(siteSocialAccount.SiteId, siteSocialAccount.FacebookPageId);
// Assert
Assert.Null(runningJob);
}
private async Task ReferenceEmployersAsync()
{
if (RunningJobs.Contains(nameof(ReferenceEmployers)))
{
return;
}
RunningJobs.Add(nameof(ReferenceEmployers));
try
{
await _OrganisationBusinessLogic.SetUniqueEmployerReferencesAsync();
}
finally
{
RunningJobs.Remove(nameof(ReferenceEmployers));
}
}
public async Task UpdateUsersAsync(string filePath)
{
if (RunningJobs.Contains(nameof(UpdateUsers)))
{
return;
}
RunningJobs.Add(nameof(UpdateUsers));
try
{
var users = await _SharedBusinessLogic.DataRepository.GetAll <User>().ToListAsync();
var records = users.Where(u => !_authorisationBusinessLogic.IsAdministrator(u))
.OrderBy(u => u.Lastname)
.Select(
u => new
{
u.Firstname,
u.Lastname,
u.JobTitle,
u.EmailAddress,
u.ContactFirstName,
u.ContactLastName,
u.ContactJobTitle,
u.ContactEmailAddress,
u.ContactPhoneNumber,
u.ContactOrganisation,
u.EmailVerifySendDate,
u.EmailVerifiedDate,
VerifyUrl = u.GetVerifyUrl(),
PasswordResetUrl = u.GetPasswordResetUrl(),
u.Status,
u.StatusDate,
u.StatusDetails,
u.Created
})
.ToList();
await Extensions.SaveCSVAsync(_SharedBusinessLogic.FileRepository, records, filePath);
}
finally
{
RunningJobs.Remove(nameof(UpdateUsers));
}
}
/// <summary>
/// Start any pending jobs as soon as we have capacity.
/// </summary>
private void ProcessWaitingJobs()
{
Job pendingJob = null;
if (CanProcessJob())
{
lock (lockObject)
{
if (CanProcessJob())
{
PendingJobQueue.TryDequeue(out pendingJob);
pendingJob.Status = JobStatus.Running;
JobRepository.Save(pendingJob);
RunningJobs.TryAdd(pendingJob, Scraper.ScrapeAsync(pendingJob.Url, pendingJob.Selectors));
}
}
}
}
public async Task UpdateOrganisationsAsync(string filePath)
{
if (RunningJobs.Contains(nameof(UpdateOrganisationsAsync)))
{
return;
}
RunningJobs.Add(nameof(UpdateOrganisationsAsync));
try
{
await WriteRecordsPerYearAsync(
filePath,
_OrganisationBusinessLogic.GetOrganisationFileModelByYearAsync);
}
finally
{
RunningJobs.Remove(nameof(UpdateOrganisationsAsync));
}
}
public async Task UpdateScopesAsync(string filePath)
{
if (RunningJobs.Contains(nameof(UpdateScopes)))
{
return;
}
RunningJobs.Add(nameof(UpdateScopes));
try
{
await WriteRecordsPerYearAsync(
filePath,
year => Task.FromResult(_ScopeBusinessLogic.GetScopesFileModelByYear(year).ToList()))
.ConfigureAwait(false);
}
finally
{
RunningJobs.Remove(nameof(UpdateScopes));
}
}
private async Task UpdateOrganisationLateSubmissionsAsync(string filePath, ILogger log)
{
var callingMethodName = nameof(UpdateOrganisationLateSubmissions);
if (RunningJobs.Contains(callingMethodName))
{
return;
}
RunningJobs.Add(callingMethodName);
try
{
var records = _SubmissionBusinessLogic.GetLateSubmissions();
await Extensions.SaveCSVAsync(_SharedBusinessLogic.FileRepository, records, filePath);
}
finally
{
RunningJobs.Remove(callingMethodName);
}
}
public void ShouldGetRunningJob()
{
// Arrange
RunningJobs runningJobs = new RunningJobs();
var scheduleManagerMock = new Mock <IScheduleJobManager>();
var siteSocialAccount = new SiteSocialAccount {
SiteId = 10000, FacebookPageId = "123"
};
Action <TriggerBuilder> configuerTriggerAction = t => { };
// Act
runningJobs.Schedule <TestJob>(scheduleManagerMock.Object, siteSocialAccount, configuerTriggerAction);
var runningJob = runningJobs.Get <TestJob>(siteSocialAccount.SiteId, siteSocialAccount.FacebookPageId);
// Assert
Assert.NotNull(runningJob);
Assert.Equal(10000, runningJob.SiteId);
Assert.Equal("123", runningJob.OriginalAccountId);
Assert.Equal("TestJob - SiteId(10000) - OriginalId(123)", runningJob.JobKey);
Assert.Equal("TestJob", runningJob.JobGroup);
}
public void ShouldUpdateLastScheduleTime()
{
// Arrange
RunningJobs runningJobs = new RunningJobs();
var scheduleManagerMock = new Mock <IScheduleJobManager>();
var siteSocialAccount = new SiteSocialAccount {
SiteId = 10000, FacebookPageId = "123"
};
Action <TriggerBuilder> configuerTriggerAction = t => { };
// Act
runningJobs.Schedule <TestJob>(scheduleManagerMock.Object, siteSocialAccount, configuerTriggerAction);
DateTime firstScheduleTime = runningJobs.Get <TestJob>(siteSocialAccount.SiteId, siteSocialAccount.FacebookPageId).LastScheduleTime;
Thread.Sleep(100);
runningJobs.Schedule <TestJob>(scheduleManagerMock.Object, siteSocialAccount, configuerTriggerAction);
DateTime secondScheduleTime = runningJobs.Get <TestJob>(siteSocialAccount.SiteId, siteSocialAccount.FacebookPageId).LastScheduleTime;
// Assert
Assert.NotEqual(firstScheduleTime, secondScheduleTime);
Assert.True(secondScheduleTime > firstScheduleTime);
}
public async Task UpdateUsersToSendInfoAsync(string filePath)
{
if (RunningJobs.Contains(nameof(UpdateUsersToSendInfo)))
{
return;
}
RunningJobs.Add(nameof(UpdateUsersToSendInfo));
try
{
var users = await _SharedBusinessLogic.DataRepository.GetAll <User>().Where(user =>
user.Status == UserStatuses.Active &&
user.UserSettings.Any(us =>
us.Key == UserSettingKeys.SendUpdates && us.Value.ToLower() == "true"))
.ToListAsync();
var records = users.Select(
u => new
{
u.Firstname,
u.Lastname,
u.JobTitle,
u.EmailAddress,
u.ContactFirstName,
u.ContactLastName,
u.ContactJobTitle,
u.ContactEmailAddress,
u.ContactPhoneNumber,
u.ContactOrganisation
})
.ToList();
await Extensions.SaveCSVAsync(_SharedBusinessLogic.FileRepository, records, filePath);
}
finally
{
RunningJobs.Remove(nameof(UpdateUsersToSendInfo));
}
}
//Ensure all organisations have a unique employer reference
public async Task ReferenceEmployers([TimerTrigger("01:00:00:00")] TimerInfo timer, ILogger log)
{
try
{
if (RunningJobs.Contains(nameof(DnBImportAsync)))
{
return;
}
await ReferenceEmployersAsync();
log.LogDebug($"Executed {nameof(ReferenceEmployers)} successfully");
}
catch (Exception ex)
{
var message = $"Failed webjob ({nameof(ReferenceEmployers)}):{ex.Message}:{ex.GetDetailsText()}";
//Send Email to GEO reporting errors
await _Messenger.SendGeoMessageAsync("GPG - WEBJOBS ERROR", message);
//Rethrow the error
throw;
}
}
public void ShouldStopTimeoutRunningJobs()
{
// Arrange
RunningJobs runningJobs = new RunningJobs();
var scheduleManagerMock = new Mock <IScheduleJobManager>();
var siteSocialAccount = new SiteSocialAccount {
SiteId = 10000, FacebookPageId = "123"
};
Action <TriggerBuilder> configuerTriggerAction = t => { };
var schedulerMock = new Mock <IScheduler>();
schedulerMock.Setup(t => t.GetJobDetail(It.IsAny <JobKey>())).Returns(new Mock <IJobDetail>().Object);
// Act
runningJobs.Schedule <TestJob>(scheduleManagerMock.Object, siteSocialAccount, configuerTriggerAction);
var runningJob = runningJobs.Get <TestJob>(siteSocialAccount.SiteId, siteSocialAccount.FacebookPageId);
runningJob.LastScheduleTime = DateTime.UtcNow.AddSeconds(-301);
runningJobs.StopTimeoutJobs(schedulerMock.Object);
// Assert
schedulerMock.Verify(t => t.DeleteJob(It.Is <JobKey>(r => r.Name == "TestJob - SiteId(10000) - OriginalId(123)")));
Assert.False(runningJobs.IsRunning <TestJob>(10000, "123"));
}
//Update GPG download file
public async Task UpdateDownloadFilesAsync(ILogger log, string userEmail = null, bool force = false)
{
if (RunningJobs.Contains(nameof(UpdateDownloadFiles)))
{
return;
}
try
{
var returnYears = _SharedBusinessLogic.DataRepository.GetAll <Return>()
.Where(r => r.Status == ReturnStatuses.Submitted)
.Select(r => r.AccountingDate.Year)
.Distinct()
.ToList();
//Get the downloads location
var downloadsLocation = _SharedBusinessLogic.SharedOptions.DownloadsLocation;
//Ensure we have a directory
if (!await _SharedBusinessLogic.FileRepository.GetDirectoryExistsAsync(downloadsLocation))
{
await _SharedBusinessLogic.FileRepository.CreateDirectoryAsync(downloadsLocation);
}
foreach (var year in returnYears)
{
//If another server is already in process of creating a file then skip
var downloadFilePattern = $"GPGData_{year}-{year + 1}.csv";
var files = await _SharedBusinessLogic.FileRepository.GetFilesAsync(downloadsLocation,
downloadFilePattern);
var oldDownloadFilePath = files.FirstOrDefault();
//Skip if the file already exists and is newer than 1 hour or older than 1 year
if (oldDownloadFilePath != null && !force)
{
var lastWriteTime =
await _SharedBusinessLogic.FileRepository.GetLastWriteTimeAsync(oldDownloadFilePath);
if (lastWriteTime.AddHours(1) >= VirtualDateTime.Now ||
lastWriteTime.AddYears(2) <= VirtualDateTime.Now)
{
continue;
}
}
var returns = await _SharedBusinessLogic.DataRepository.GetAll <Return>().Where(r =>
r.AccountingDate.Year == year &&
r.Status == ReturnStatuses.Submitted &&
r.Organisation.Status == OrganisationStatuses.Active)
.ToListAsync();
returns.RemoveAll(r =>
r.Organisation.OrganisationName.StartsWithI(_SharedBusinessLogic.SharedOptions.TestPrefix));
var downloadData = returns.ToList()
.Select(r => DownloadResult.Create(r))
.OrderBy(d => d.EmployerName)
.ToList();
var newFilePath =
_SharedBusinessLogic.FileRepository.GetFullPath(Path.Combine(downloadsLocation,
$"GPGData_{year}-{year + 1}.csv"));
try
{
if (downloadData.Any())
{
await Extensions.SaveCSVAsync(_SharedBusinessLogic.FileRepository, downloadData,
newFilePath, oldDownloadFilePath);
}
else if (!string.IsNullOrWhiteSpace(oldDownloadFilePath))
{
await _SharedBusinessLogic.FileRepository.DeleteFileAsync(oldDownloadFilePath);
}
}
catch (Exception ex)
{
log.LogError(ex, ex.Message);
}
}
if (force && !string.IsNullOrWhiteSpace(userEmail))
{
try
{
await _Messenger.SendMessageAsync(
"UpdateDownloadFiles complete",
userEmail,
"The update of the download files completed successfully.");
}
catch (Exception ex)
{
log.LogError(ex, ex.Message);
}
}
}
finally
{
RunningJobs.Remove(nameof(UpdateDownloadFiles));
}
}
private RunningJob FindRunningJob(OpenJob openJob)
{
return(RunningJobs.FirstOrDefault(runningJob => openJob.PipelineId == runningJob.PipelineId));
}
private async Task FixOrganisationsNamesAsync(ILogger log, string userEmail, string comment)
{
if (RunningJobs.Contains(nameof(FixOrganisationsNamesAsync)))
{
return;
}
RunningJobs.Add(nameof(FixOrganisationsNamesAsync));
try
{
var orgs = await _SharedBusinessLogic.DataRepository.GetAll <Organisation>().ToListAsync();
var count = 0;
var i = 0;
foreach (var org in orgs)
{
i++;
var names = org.OrganisationNames.OrderBy(n => n.Created).ToList();
var changed = false;
;
while (names.Count > 1 && names[1].Name
.EqualsI(names[0].Name.Replace(" LTD.", " LIMITED").Replace(" Ltd", " Limited")))
{
await _ManualChangeLog.WriteAsync(
new ManualChangeLogModel(
nameof(FixOrganisationsNamesAsync),
ManualActions.Delete,
userEmail,
nameof(Organisation.EmployerReference),
org.EmployerReference,
null,
JsonConvert.SerializeObject(
new { names[0].Name, names[0].Source, names[0].Created, names[0].OrganisationId }),
null,
comment));
names[1].Created = names[0].Created;
_SharedBusinessLogic.DataRepository.Delete(names[0]);
names.RemoveAt(0);
changed = true;
}
;
if (names.Count > 0)
{
var newValue = names[names.Count - 1].Name;
if (org.OrganisationName != newValue)
{
org.OrganisationName = newValue;
await _ManualChangeLog.WriteAsync(
new ManualChangeLogModel(
nameof(FixOrganisationsNamesAsync),
ManualActions.Update,
userEmail,
nameof(Organisation.EmployerReference),
org.EmployerReference,
nameof(org.OrganisationName),
org.OrganisationName,
newValue,
comment));
changed = true;
}
}
if (changed)
{
count++;
await _SharedBusinessLogic.DataRepository.SaveChangesAsync();
}
}
log.LogDebug($"Executed {nameof(FixOrganisationsNamesAsync)} successfully and deleted {count} names");
}
finally
{
RunningJobs.Remove(nameof(FixOrganisationsNamesAsync));
}
}
public async Task DnBImportAsync(ILogger log, long currentUserId)
{
if (RunningJobs.Contains(nameof(CompaniesHouseCheck)) ||
RunningJobs.Contains(nameof(DnBImportAsync)))
{
return;
}
RunningJobs.Add(nameof(DnBImportAsync));
string userEmail = null;
string error = null;
var startTime = VirtualDateTime.Now;
var totalChanges = 0;
var totalInserts = 0;
try
{
#region Load and Prechecks
//Load the D&B records
var dnbOrgsPaths =
await _SharedBusinessLogic.FileRepository.GetFilesAsync(_SharedBusinessLogic.SharedOptions.DataPath,
Filenames.DnBOrganisations());
var dnbOrgsPath = dnbOrgsPaths.OrderByDescending(f => f).FirstOrDefault();
if (string.IsNullOrEmpty(dnbOrgsPath))
{
return;
}
if (!await _SharedBusinessLogic.FileRepository.GetFileExistsAsync(dnbOrgsPath))
{
throw new Exception("Could not find " + dnbOrgsPath);
}
var AllDnBOrgs = await _SharedBusinessLogic.FileRepository.ReadCSVAsync <DnBOrgsModel>(dnbOrgsPath);
if (!AllDnBOrgs.Any())
{
log.LogWarning($"No records found in '{dnbOrgsPath}'");
return;
}
AllDnBOrgs = AllDnBOrgs.OrderBy(o => o.OrganisationName).ToList();
//Check for duplicate DUNS
var count = AllDnBOrgs.Count() - AllDnBOrgs.Select(o => o.DUNSNumber).Distinct().Count();
if (count > 0)
{
throw new Exception($"There are {count} duplicate DUNS numbers detected");
}
//Check for no addresses
count = AllDnBOrgs.Count(o => !o.IsValidAddress());
if (count > 0)
{
throw new Exception(
$"There are {count} organisations with no address detected (i.e., no AddressLine1, AddressLine2, PostalCode, and PoBox).");
}
//Check for no organisation name
count = AllDnBOrgs.Count(o => string.IsNullOrWhiteSpace(o.OrganisationName));
if (count > 0)
{
throw new Exception($"There are {count} organisations with no OrganisationName detected.");
}
//Check for duplicate employer references
var allEmployerReferenceCount = AllDnBOrgs.Count(o => !string.IsNullOrWhiteSpace(o.EmployerReference));
var employerReferences = new SortedSet <string>(
AllDnBOrgs.Where(o => !string.IsNullOrWhiteSpace(o.EmployerReference))
.Select(o => o.EmployerReference).Distinct());
count = allEmployerReferenceCount - employerReferences.Count;
if (count > 0)
{
throw new Exception($"There are {count} duplicate EmployerReferences detected");
}
//Check companies have been updated
count = AllDnBOrgs.Count(
o => !string.IsNullOrWhiteSpace(o.CompanyNumber) &&
o.DateOfCessation == null &&
(o.StatusCheckedDate == null ||
o.StatusCheckedDate.Value.AddMonths(1) < VirtualDateTime.Now));
if (count > 0)
{
throw new Exception(
$"There are {count} active companies who have not been checked with companies house within the last month");
}
//Fix Company Number
Parallel.ForEach(
AllDnBOrgs.Where(o => !string.IsNullOrWhiteSpace(o.CompanyNumber)),
dnbOrg =>
{
if (dnbOrg.CompanyNumber.IsNumber())
{
dnbOrg.CompanyNumber = dnbOrg.CompanyNumber.PadLeft(8, '0');
}
});
//Check for duplicate company numbers
var companyNumbers =
AllDnBOrgs.Where(o => !string.IsNullOrWhiteSpace(o.CompanyNumber)).Select(o => o.CompanyNumber);
count = companyNumbers.Count() - companyNumbers.Distinct().Count();
if (count > 0)
{
throw new Exception($"There are {count} duplicate CompanyNumbers detected");
}
//Get the current users email address
var user = await _SharedBusinessLogic.DataRepository.GetAll <User>()
.FirstOrDefaultAsync(u => u.UserId == currentUserId);
userEmail = user?.EmailAddress;
//Count records requiring import
count = AllDnBOrgs.Count(
o => !o.GetIsDissolved() &&
(o.ImportedDate == null || string.IsNullOrWhiteSpace(o.CompanyNumber) ||
o.ImportedDate < o.StatusCheckedDate));
if (count == 0)
{
return;
}
var dbOrgs = _SharedBusinessLogic.DataRepository.GetAll <Organisation>().ToList();
#endregion
//Set all existing org employer references
await ReferenceEmployersAsync();
#region Set all existing org DUNS
var dnbOrgs = AllDnBOrgs
.Where(o => o.OrganisationId > 0 && string.IsNullOrWhiteSpace(o.EmployerReference))
.ToList();
if (dnbOrgs.Count > 0)
{
foreach (var dnbOrg in dnbOrgs)
{
var org = dbOrgs.FirstOrDefault(o => o.OrganisationId == dnbOrg.OrganisationId);
if (org == null)
{
if (!_SharedBusinessLogic.SharedOptions.IsProduction())
{
continue;
}
throw new Exception($"OrganisationId:{dnbOrg.OrganisationId} does not exist in database");
}
if (!string.IsNullOrWhiteSpace(org.DUNSNumber))
{
continue;
}
org.DUNSNumber = dnbOrg.DUNSNumber;
dnbOrg.OrganisationId = null;
}
await _SharedBusinessLogic.DataRepository.SaveChangesAsync();
dbOrgs = await _SharedBusinessLogic.DataRepository.GetAll <Organisation>().ToListAsync();
await _SharedBusinessLogic.FileRepository.SaveCSVAsync(AllDnBOrgs, dnbOrgsPath);
AllDnBOrgs = await _SharedBusinessLogic.FileRepository.ReadCSVAsync <DnBOrgsModel>(dnbOrgsPath);
AllDnBOrgs = AllDnBOrgs.OrderBy(o => o.OrganisationName).ToList();
}
#endregion
var allSicCodes = await _SharedBusinessLogic.DataRepository.GetAll <SicCode>().ToListAsync();
dnbOrgs = AllDnBOrgs.Where(o => o.ImportedDate == null || o.ImportedDate < o.StatusCheckedDate)
.ToList();
while (dnbOrgs.Count > 0)
{
var allBadSicCodes = new ConcurrentBag <OrganisationSicCode>();
var c = 0;
var dbChanges = 0;
var dnbChanges = 0;
foreach (var dnbOrg in dnbOrgs)
{
//Only do 100 records at a time
if (c > 100)
{
break;
}
var dbChanged = false;
var dbOrg = dbOrgs.FirstOrDefault(o => o.DUNSNumber == dnbOrg.DUNSNumber);
var dataSource = string.IsNullOrWhiteSpace(dnbOrg.NameSource) ? "D&B" : dnbOrg.NameSource;
var orgName = new OrganisationName
{
Name = dnbOrg.OrganisationName.Left(100), Source = dataSource
};
if (dbOrg == null)
{
dbOrg = string.IsNullOrWhiteSpace(dnbOrg.CompanyNumber)
? null
: dbOrgs.FirstOrDefault(o => o.CompanyNumber == dnbOrg.CompanyNumber);
if (dbOrg != null)
{
dbOrg.DUNSNumber = dnbOrg.DUNSNumber;
}
else
{
dbOrg = new Organisation
{
DUNSNumber = dnbOrg.DUNSNumber,
EmployerReference = dnbOrg.EmployerReference,
OrganisationName = orgName.Name,
CompanyNumber = string.IsNullOrWhiteSpace(dnbOrg.CompanyNumber)
? null
: dnbOrg.CompanyNumber,
SectorType = dnbOrg.SectorType,
DateOfCessation = dnbOrg.DateOfCessation
};
dbOrg.OrganisationNames.Add(orgName);
//Create a presumed in-scope for current year
var newScope = new OrganisationScope
{
Organisation = dbOrg,
ScopeStatus = ScopeStatuses.PresumedInScope,
ScopeStatusDate = VirtualDateTime.Now,
Status = ScopeRowStatuses.Active,
SnapshotDate = _snapshotDateHelper.GetSnapshotDate(dbOrg.SectorType)
};
_SharedBusinessLogic.DataRepository.Insert(newScope);
dbOrg.OrganisationScopes.Add(newScope);
//Create a presumed out-of-scope for previous year
var oldScope = new OrganisationScope
{
Organisation = dbOrg,
ScopeStatus = ScopeStatuses.PresumedOutOfScope,
ScopeStatusDate = VirtualDateTime.Now,
Status = ScopeRowStatuses.Active,
SnapshotDate = newScope.SnapshotDate.AddYears(-1)
};
_SharedBusinessLogic.DataRepository.Insert(oldScope);
dbOrg.OrganisationScopes.Add(oldScope);
dbOrg.SetStatus(OrganisationStatuses.Active, currentUserId, "Imported from D&B");
}
}
//Skip dissolved companies
else if (_OrganisationBusinessLogic.GetOrganisationWasDissolvedBeforeCurrentAccountingYear(dbOrg))
{
dnbOrg.ImportedDate = VirtualDateTime.Now;
dnbChanges++;
continue;
}
else if (dbOrg.OrganisationName != orgName.Name)
{
var oldOrgName = dbOrg.GetLatestName();
if (oldOrgName == null ||
_SharedBusinessLogic.SourceComparer.CanReplace(orgName.Source, oldOrgName.Source))
{
dbOrg.OrganisationName = orgName.Name;
dbOrg.OrganisationNames.Add(orgName);
dbChanged = true;
}
}
//Ensure D&B gas an organisationID
if (dnbOrg.OrganisationId == null || dnbOrg.OrganisationId.Value == 0)
{
dnbOrg.OrganisationId = dbOrg.OrganisationId;
dnbChanges++;
}
//Add the cessation date
if (dbOrg.DateOfCessation == null && dbOrg.DateOfCessation != dnbOrg.DateOfCessation)
{
dbOrg.DateOfCessation = dnbOrg.DateOfCessation;
dbChanged = true;
}
//Set the employer reference
if (string.IsNullOrWhiteSpace(dbOrg.EmployerReference))
{
string employerReference;
do
{
employerReference = _OrganisationBusinessLogic.GenerateEmployerReference();
} while (employerReferences.Contains(employerReference));
dbOrg.EmployerReference = employerReference;
employerReferences.Add(employerReference);
dbChanged = true;
}
if (dnbOrg.EmployerReference != dbOrg.EmployerReference)
{
dnbOrg.EmployerReference = dbOrg.EmployerReference;
dnbChanges++;
}
//Add the new address
var fullAddress = dnbOrg.GetAddress();
var newAddress = dbOrg.LatestAddress;
//add the address if there isnt one
dataSource = string.IsNullOrWhiteSpace(dnbOrg.AddressSource) ? "D&B" : dnbOrg.AddressSource;
if (newAddress == null ||
!newAddress.GetAddressString().EqualsI(fullAddress) &&
_SharedBusinessLogic.SourceComparer.CanReplace(dataSource, newAddress.Source))
{
var statusDate = VirtualDateTime.Now;
newAddress = new OrganisationAddress();
newAddress.Organisation = dbOrg;
newAddress.CreatedByUserId = currentUserId;
newAddress.Address1 = dnbOrg.AddressLine1;
newAddress.Address2 = dnbOrg.AddressLine2;
newAddress.Address3 = dnbOrg.AddressLine3;
newAddress.County = dnbOrg.County;
newAddress.Country = dnbOrg.Country;
newAddress.TownCity = dnbOrg.City;
newAddress.PostCode = dnbOrg.PostalCode;
newAddress.PoBox = dnbOrg.PoBox;
newAddress.Source = dataSource;
newAddress.SetStatus(AddressStatuses.Active, currentUserId, "Imported from D&B");
if (dbOrg.LatestAddress != null)
{
dbOrg.LatestAddress.SetStatus(AddressStatuses.Retired, currentUserId,
$"Replaced by {newAddress.Source}");
}
}
//Update the sic codes
var newCodeIds = dnbOrg.GetSicCodesIds();
var newCodesList = dnbOrg.GetSicCodesIds().ToList();
for (var i = 0; i < newCodesList.Count; i++)
{
var code = newCodesList[i];
if (code <= 0)
{
continue;
}
var sicCode = allSicCodes.FirstOrDefault(sic => sic.SicCodeId == code);
if (sicCode != null)
{
continue;
}
sicCode = allSicCodes.FirstOrDefault(
sic => sic.SicCodeId == code * 10 && sic.Description.EqualsI(dnbOrg.SicDescription));
if (sicCode != null)
{
newCodesList[i] = sicCode.SicCodeId;
}
}
newCodeIds = new SortedSet <int>(newCodesList);
var newCodes = new List <OrganisationSicCode>();
var oldCodes = dbOrg.GetLatestSicCodes().ToList();
var oldSicSource = dbOrg.GetLatestSicSource();
var oldCodeIds = oldCodes.Select(s => s.SicCodeId);
if (dbOrg.SectorType == SectorTypes.Public)
{
newCodeIds.Add(1);
}
if (!_SharedBusinessLogic.SharedOptions.IsProduction())
{
Debug.WriteLine(
$"OLD:{oldCodes.Select(s => s.SicCodeId).ToDelimitedString()} NEW:{newCodeIds.ToDelimitedString()}");
}
dataSource = string.IsNullOrWhiteSpace(dnbOrg.SicSource) ? "D&B" : dnbOrg.SicSource;
if (!newCodeIds.SetEquals(oldCodeIds) &&
_SharedBusinessLogic.SourceComparer.CanReplace(dataSource, oldSicSource))
{
foreach (var code in newCodeIds)
{
if (code <= 0)
{
continue;
}
var sicCode = allSicCodes.FirstOrDefault(sic => sic.SicCodeId == code);
var newSic = new OrganisationSicCode
{
Organisation = dbOrg, SicCodeId = code, Source = dataSource
};
if (sicCode == null)
{
allBadSicCodes.Add(newSic);
continue;
}
newCodes.Add(newSic);
}
if (newCodes.Any())
{
//Add new codes only
foreach (var newSic in newCodes)
{
dbOrg.OrganisationSicCodes.Add(newSic);
dbChanged = true;
}
//Retire the old codes
foreach (var oldSic in oldCodes)
{
oldSic.Retired = VirtualDateTime.Now;
dbChanged = true;
}
}
}
await _SharedBusinessLogic.DataRepository.BeginTransactionAsync(
async() =>
{
try
{
//Save the name, Sic, EmployerReference, DateOfCessasion changes
if (dbChanged)
{
await _SharedBusinessLogic.DataRepository.SaveChangesAsync();
}
//Save the changes
dnbOrg.ImportedDate = VirtualDateTime.Now;
dnbChanges++;
var insert = false;
if (dbOrg.OrganisationId == 0)
{
_SharedBusinessLogic.DataRepository.Insert(dbOrg);
await _SharedBusinessLogic.DataRepository.SaveChangesAsync();
dbChanged = true;
insert = true;
}
if (newAddress != null && newAddress.AddressId == 0)
{
dbOrg.OrganisationAddresses.Add(newAddress);
dbOrg.LatestAddress = newAddress;
await _SharedBusinessLogic.DataRepository.SaveChangesAsync();
dbChanged = true;
}
if (dbChanged)
{
dbChanges++;
_SharedBusinessLogic.DataRepository.CommitTransaction();
totalChanges++;
if (insert)
{
totalInserts++;
}
//Add or remove this organisation to/from the search index
await SearchBusinessLogic.UpdateSearchIndexAsync(dbOrg);
}
}
catch
{
_SharedBusinessLogic.DataRepository.RollbackTransaction();
}
});
c++;
}
//Reload all the changes
if (dbChanges > 0)
{
dbOrgs = await _SharedBusinessLogic.DataRepository.GetAll <Organisation>().ToListAsync();
}
//Save the D&B records
if (dnbChanges > 0)
{
await _SharedBusinessLogic.FileRepository.SaveCSVAsync(AllDnBOrgs, dnbOrgsPath);
AllDnBOrgs = await _SharedBusinessLogic.FileRepository.ReadCSVAsync <DnBOrgsModel>(dnbOrgsPath);
AllDnBOrgs = AllDnBOrgs.OrderBy(o => o.OrganisationName).ToList();
dnbOrgs = AllDnBOrgs.Where(o => o.ImportedDate == null || o.ImportedDate < o.StatusCheckedDate)
.ToList();
}
//Save the bad sic codes
if (allBadSicCodes.Count > 0)
{
//Create the logging tasks
var badSicLoggingtasks = new List <Task>();
allBadSicCodes.ForEach(
bsc => badSicLoggingtasks.Add(
_BadSicLog.WriteAsync(
new BadSicLogModel
{
OrganisationId = bsc.Organisation.OrganisationId,
OrganisationName = bsc.Organisation.OrganisationName,
SicCode = bsc.SicCodeId,
Source = bsc.Source
})));
//Wait for all the logging tasks to complete
await Task.WhenAll(badSicLoggingtasks);
}
}
}
catch (Exception ex)
{
error = ex.Message;
throw;
}
finally
{
if (!string.IsNullOrWhiteSpace(userEmail))
{
var endTime = VirtualDateTime.Now;
var duration = endTime - startTime;
try
{
if (!string.IsNullOrWhiteSpace(error))
{
await _Messenger.SendMessageAsync(
"D&B Import Failed",
userEmail,
$"The D&B import failed at {endTime} after {duration.ToFriendly()}.\nChanged {totalChanges} organisations including {totalInserts} new.\n\nERROR:{error}");
}
else if (totalChanges == 0)
{
await _Messenger.SendMessageAsync(
"D&B Import Complete",
userEmail,
"The D&B import process completed successfully with no records requiring import.");
}
else
{
await _Messenger.SendMessageAsync(
"D&B Import Complete",
userEmail,
$"The D&B import process completed successfully at {endTime} after {duration.ToFriendly()}.\nChanged {totalChanges} organisations including {totalInserts} new.");
}
}
catch (Exception ex)
{
log.LogError(ex, ex.Message);
}
}
RunningJobs.Remove(nameof(DnBImportAsync));
}
}
