/// <summary>
/// Gets the access token.
/// </summary>
/// <returns></returns>
/// <exception cref="ApplicationException">Couldn't get access token. Error: " + tokenResponse.Error</exception>
public async Task <string> GetAccessTokenAsync()
{
if (string.IsNullOrEmpty(_accessToken))
{
using (var tokenClient = new TokenClient(TokenEndpoint, ClientId, ClientSecret, AuthenticationStyle.PostValues))
{
logger.Info($"Token client {TokenEndpoint} called with client {ClientId}.");
//This is call to the token endpoint with the parameters that are set
var tokenResponse = await tokenClient.RequestResourceOwnerPasswordAsync(Username, Password, Scopes, AdditionalParameters);
//var tokenResponse = await tokenClient.RequestClientCredentialsAsync("offline_access");
if (tokenResponse.IsError)
{
logger.Info($"Token client {TokenEndpoint} called with client {ClientId} failed with error {tokenResponse.Error}.");
throw new ApplicationException("Couldn't get access token. Error: " + tokenResponse.Error);
}
_accessToken = tokenResponse.AccessToken;
}
}
return(_accessToken);
}
private Policy GetCircuitBreaker(string dependency)
{
return(Policy
.Handle <Exception>()
.CircuitBreaker(
exceptionsAllowedBeforeBreaking: strategy.AllowedFaults,
durationOfBreak: strategy.Breaktime,
onBreak: (ex, breakDelay) => logger.ErrorJustLogIt($"{nameof(TolerancePolicy)}:Circuit-Breaker logging: Broken : {dependency}: Breaking the circuit for {breakDelay}, failure-{ex.Message}", ex),
onReset: () => logger.Info($"{nameof(TolerancePolicy)}:Circuit-Breaker logging: {dependency}: Call succeeded. Closed the circuit again!"),
onHalfOpen: () => logger.Warn($"{nameof(TolerancePolicy)}:Circuit-Breaker logging: {dependency}: Half-open: Next call is a trial!")));
}
public async Task InvokeAsync(HttpContext context, IApplicationLogger logger)
{
logger
.Info($"Execution started [{context.Request.RouteValues["controller"]}.{context.Request.RouteValues["action"]} {context.Request.Method}]")
.Write();
await _next.Invoke(context);
logger
.Info($"Execution finished [{context.Request.RouteValues["controller"]}.{context.Request.RouteValues["action"]} {context.Request.Method}]")
.Write();
}
public async Task DeleteOrphanedAvsAsync()
{
logger.Info("Getting orphaned apprenticeship vacancies");
var orphanedApprenticeshipVacancies = await apprenticeshipVacancyRepository.GetOrphanedApprenticeshipVacanciesAsync();
await auditService.AuditAsync($"Got {orphanedApprenticeshipVacancies.Count()} orphaned apprenticeship vacancies");
foreach (OrphanedVacancySummary o in orphanedApprenticeshipVacancies)
{
await auditService.AuditAsync($"Got orphaned apprenticeship deleting vacancies id= {o.Id} published = {o.PublicationDate.ToString()} title = {o.Title} vacancyId = {o.VacancyId}");
await apprenticeshipVacancyRepository.DeleteByIdAsync(o.Id);
}
logger.Info("Completed deleting orphaned apprenticeship vacancies");
}
public override void UpdateIndex(string indexName, IEnumerable <IDocument> documents)
{
var activeIndex = string.Empty;
var measure = Stopwatch.StartNew();
try
{
if (!string.IsNullOrEmpty(indexName) && index.StartsWith(indexName, StringComparison.OrdinalIgnoreCase))
{
activeIndex = index;
var jpIndexDoc = documents.ConvertToJobProfileIndex(jobProfileIndexEnhancer, applicationLogger);
//Requires a deep copy to ensure the enumerable is not executed again on a non-ui thread which sitefinity relies upon!!!
var copy = mapper.Map <IEnumerable <JobProfileIndex> >(jpIndexDoc);
asyncHelper.Synchronise(() => searchService?.PopulateIndexAsync(copy));
}
else
{
activeIndex = indexName;
base.UpdateIndex(indexName, documents);
}
}
catch (Exception exception)
{
applicationLogger.Error($" Method - {MethodBase.GetCurrentMethod().Name} on index {activeIndex} failed with an exception", exception);
}
applicationLogger.Info($"Took {measure.Elapsed} to complete the indexing on {activeIndex}");
}
public async Task ExecuteAsync(ProjectedVacancyDetails myQueueItem)
{
List <PublishedAV> publishedVacancies = new List <PublishedAV>();
sitefinityTokenClient.SetAccessToken(myQueueItem.AccessToken);
await apprenticeshipVacancyRepository.DeleteExistingAsync(myQueueItem.SocCode);
logger.Info($"Deleted all vacancies for soc code {myQueueItem.SocCode}, ready for publishing {myQueueItem.Vacancies.Count()} vacancies.");
foreach (var vacancy in myQueueItem.Vacancies)
{
var urlName = await apprenticeshipVacancyRepository.PublishAsync(vacancy, myQueueItem.SocMappingId);
publishedVacancies.Add(new PublishedAV
{
UrlName = urlName,
VacancyReference = vacancy.VacancyReference,
Title = vacancy.Title,
VacancyUrl = vacancy.VacancyUrl
});
}
output = new PublishedVacancySummary
{
SocCode = myQueueItem.SocCode,
SocMappingId = myQueueItem.SocMappingId,
Vacancies = publishedVacancies,
};
}
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
{
switch (logLevel)
{
case LogLevel.Information:
applicationLogger.Info($"{nameof(CourseSearchLogger<T>)}-{typeof(T)}-EventId:{eventId}-Formatted:{formatter(state, exception)}");
break;
case LogLevel.Warning:
applicationLogger.Warn($"{nameof(CourseSearchLogger<T>)}-EventId:{eventId}-Formatted:{formatter(state, exception)}");
break;
case LogLevel.Critical:
case LogLevel.Error:
applicationLogger.ErrorJustLogIt($"{nameof(CourseSearchLogger<T>)}-{typeof(T)}-EventId:{eventId}-Formatted:{formatter(state, exception)}", exception);
break;
case LogLevel.None:
case LogLevel.Trace:
case LogLevel.Debug:
default:
applicationLogger.Trace($"{nameof(CourseSearchLogger<T>)}-{typeof(T)}-EventId:{eventId}-Formatted:{formatter(state, exception)}");
break;
}
}
/// <summary>
/// Run the application
/// </summary>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The task to run</returns>
public async Task Run(CancellationToken cancellationToken)
{
bool isInfoEnabled = _logger.IsInfoEnabled;
if (isInfoEnabled == true)
{
_logger.Info(LogResource.Starting);
}
await _application
.Run(cancellationToken)
.ConfigureAwait(Await.Default);
if (isInfoEnabled == true)
{
_logger.Info(LogResource.Finished);
}
}
public void Update(DynamicContent entity, string changeComment)
{
if (entity == null)
{
throw new ArgumentNullException(nameof(entity));
}
// Set a transaction name and get the version manager
var transactionName = $"{entity.GetType().Name}-{DateTime.Now.Ticks}";
applicationLogger.Info($"Updating entity with transaction name {transactionName} for {entity.UrlName}");
var versionManager = VersionManager.GetManager(null, transactionName);
CreateVersion(entity, changeComment, versionManager, WorkflowStatus.Draft);
// Commit the transaction in order for the items to be actually persisted to data store
TransactionManager.CommitTransaction(transactionName);
}
public async Task <PagedOdataResult <T> > GetResult(Uri requestUri, bool shouldAudit)
{
try
{
return(await GetInternalAsync(requestUri, shouldAudit));
}
catch (UnauthorizedAccessException)
{
logger.Info($"Access denined, access token expired - will retry with new token - '{requestUri}'.");
tokenClient.SetAccessToken(string.Empty);
return(await GetInternalAsync(requestUri, shouldAudit));
}
}
public async Task DeleteExistingAsync(string SOC)
{
var existingAvCollection = await repository.GetManyAsync(av => av.SOCCode != null && av.SOCCode.SOCCode == SOC);
logger.Info($"Deleting '{existingAvCollection.Count()}' vacancies from sitefintiy for SOC '{SOC}'");
foreach (var av in existingAvCollection)
{
await repository.DeleteAsync(av);
logger.Info($"Deleted vacancy '{av.UrlName}-{av.Title}' from sitefintiy for SocCode '{av.SOCCode.SOCCode}'");
}
}
public bool DeleteVacanciesPermanently(int itemCount)
{
var lastDeleteCall = false;
using (var recycleBinItemsManager = RecycleBinManagerFactory.GetManager())
{
SystemManager.RunWithElevatedPrivilege(d =>
{
var recycleBinItems = recycleBinItemsManager
.GetRecycleBinItems()
.Where(di => di.DeletedItemTypeName.Equals(ApprenticeVacancyDeleteTypeName))
.Take(itemCount)
.ToList();
lastDeleteCall = itemCount > recycleBinItems.Count;
var providerName = DynamicModuleManager.GetDefaultProviderName(DynamicTypes.JobProfileModuleName);
var dynamicModuleContentType = TypeResolutionService.ResolveType(ApprenticeVacancyDeleteTypeName);
using (var dynamicModuleManager = DynamicModuleManager.GetManager(providerName))
{
foreach (var recycleBinItem in recycleBinItems)
{
try
{
var dataItem = dynamicModuleManager.GetItem(dynamicModuleContentType, recycleBinItem.DeletedItemId) as IRecyclableDataItem;
dynamicModuleManager.RecycleBin.PermanentlyDeleteFromRecycleBin(dataItem);
}
catch (ItemNotFoundException exception)
{
recycleBinItemsManager.Delete(recycleBinItem);
applicationLogger.Info($"Could not delete item : {recycleBinItem.DeletedItemTitle}, failed with error {exception.Message}");
recycleBinItemsManager.SaveChanges();
}
}
dynamicModuleManager.SaveChanges();
}
});
}
return(lastDeleteCall);
}
public async Task SendContentPageMessage(MicroServicesPublishingPageData contentPageData, string contentType, string actionType)
{
applicationLogger.Info($" CREATED service bus message for sitefinity event {actionType.ToUpper()} on ContentPage with Title -- {contentPageData.CanonicalName} and Id -- {contentPageData.ContentPageId.ToString()}");
var connectionStringServiceBus = configurationProvider.GetConfig <string>("DFC.Digital.ServiceBus.ConnectionString");
var topicName = configurationProvider.GetConfig <string>("DFC.Digital.ServiceBus.TopicName");
var topicClient = new TopicClient(connectionStringServiceBus, topicName);
// Send Messages
var jsonData = JsonConvert.SerializeObject(contentPageData);
try
{
applicationLogger.Info($" SENDING service bus message for sitefinity event {actionType.ToUpper()} on ContentPage with Title -- {contentPageData.CanonicalName} and Id -- {contentPageData.ContentPageId.ToString()} ");
// Message that send to the queue
var message = new Message(Encoding.UTF8.GetBytes(jsonData));
message.ContentType = "application/json";
message.Label = contentPageData.CanonicalName;
message.UserProperties.Add("Id", contentPageData.ContentPageId);
message.UserProperties.Add("ActionType", actionType);
message.UserProperties.Add("CType", contentType);
message.CorrelationId = Guid.NewGuid().ToString();
await topicClient.SendAsync(message);
applicationLogger.Info($" SENT SUCCESSFULLY service bus message for sitefinity event {actionType.ToUpper()} on ContentPage with Title -- {contentPageData.CanonicalName} with Id -- {contentPageData.ContentPageId.ToString()} and with Correlation Id -- {message.CorrelationId.ToString()}");
}
catch (Exception ex)
{
applicationLogger.Info($" FAILED service bus message for sitefinity event {actionType.ToUpper()} on ContentPage with Title -- {contentPageData.CanonicalName} with Id -- {contentPageData.ContentPageId.ToString()} has an exception \n {ex.Message} ");
}
finally
{
await topicClient.CloseAsync();
}
}
/// <summary>
/// 写入一个info等级的日志记录.
/// </summary>
/// <param name="logger">日志记录器.</param>
/// <param name="exception">要记录的异常.</param>
/// <param name="format">待格式化的消息字符串.</param>
/// <param name="args">格式化参数列表.</param>
public static void Info(this IApplicationLogger logger, Exception exception, String format, params Object[] args)
{
logger.Info(exception, String.Format(format, args));
}
internal static IEnumerable <JobProfileIndex> ConvertToJobProfileIndex(this IEnumerable <IDocument> documents, IJobProfileIndexEnhancer jobProfileIndexEnhancer, IApplicationLogger applicationLogger)
{
var measure = Stopwatch.StartNew();
Dictionary <string, JobProfileIndex> indexes = new Dictionary <string, JobProfileIndex>();
var salaryPopulation = new List <Task <JobProfileSalary> >();
var betaDocuments = documents.Where(d => Convert.ToBoolean(d.GetValue(nameof(JobProfile.IsImported)) ?? false) == false);
foreach (var item in betaDocuments)
{
//TODO: Check and confirm that the removed FilterableTitle and FilterableAlternativeTitle are no longer used.
var jobProfileIndex = new JobProfileIndex
{
IdentityField = item.IdentityField.Value?.ToString(),
UrlName = item.GetValue(nameof(JobProfileIndex.UrlName))?.ToString(),
Title = item.GetValue(nameof(JobProfileIndex.Title))?.ToString(),
AlternativeTitle = item.GetValue(nameof(JobProfileIndex.AlternativeTitle))?.ToString().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(a => a.Trim()),
Overview = item.GetValue(nameof(JobProfileIndex.Overview))?.ToString(),
JobProfileCategories = item.GetValue(nameof(JobProfileIndex.JobProfileCategories)) as IEnumerable <string>,
JobProfileSpecialism = item.GetValue(nameof(JobProfileIndex.JobProfileSpecialism)) as IEnumerable <string>,
HiddenAlternativeTitle = item.GetValue(nameof(JobProfileIndex.HiddenAlternativeTitle)) as IEnumerable <string>,
WYDDayToDayTasks = item.GetValue(nameof(JobProfileIndex.WYDDayToDayTasks))?.ToString(),
CollegeRelevantSubjects = item.GetValue(nameof(JobProfileIndex.CollegeRelevantSubjects))?.ToString(),
UniversityRelevantSubjects = item.GetValue(nameof(JobProfileIndex.UniversityRelevantSubjects))?.ToString(),
ApprenticeshipRelevantSubjects = item.GetValue(nameof(JobProfileIndex.ApprenticeshipRelevantSubjects))?.ToString(),
CareerPathAndProgression = item.GetValue(nameof(JobProfileIndex.CareerPathAndProgression))?.ToString(),
WorkingPattern = item.GetValue(nameof(JobProfileIndex.WorkingPattern)) as IEnumerable <string>,
WorkingHoursDetails = item.GetValue(nameof(JobProfileIndex.WorkingHoursDetails)) as IEnumerable <string>,
WorkingPatternDetails = item.GetValue(nameof(JobProfileIndex.WorkingPatternDetails)) as IEnumerable <string>,
MinimumHours = Convert.ToDouble(item.GetValue(nameof(JobProfileIndex.MinimumHours))),
MaximumHours = Convert.ToDouble(item.GetValue(nameof(JobProfileIndex.MaximumHours)))
};
var isSalaryOverriden = Convert.ToBoolean(item.GetValue(nameof(JobProfile.IsLMISalaryFeedOverriden)));
jobProfileIndexEnhancer.Initialise(jobProfileIndex, documents.Count() == 1);
jobProfileIndexEnhancer.PopulateRelatedFieldsWithUrl();
if (isSalaryOverriden)
{
jobProfileIndex.SalaryStarter = Convert.ToDouble(item.GetValue(nameof(JobProfile.SalaryStarter)));
jobProfileIndex.SalaryExperienced = Convert.ToDouble(item.GetValue(nameof(JobProfile.SalaryExperienced)));
}
else
{
if (!string.IsNullOrEmpty(jobProfileIndex.SocCode))
{
salaryPopulation.Add(jobProfileIndexEnhancer.PopulateSalary(jobProfileIndex.SocCode, jobProfileIndex.UrlName));
}
}
indexes.Add(jobProfileIndex.UrlName.ToLower(), jobProfileIndex);
}
var results = Task.Run(() => Task.WhenAll(salaryPopulation.ToArray())).GetAwaiter().GetResult();
foreach (var idx in indexes)
{
var item = results.SingleOrDefault(r => r.JobProfileName.Equals(idx.Key, StringComparison.InvariantCultureIgnoreCase));
if (item == null)
{
applicationLogger.Warn($"WARN: Failed to get salary for '{idx.Key}'.");
continue;
}
idx.Value.SalaryStarter = item.StarterSalary;
idx.Value.SalaryExperienced = item.SalaryExperienced;
}
applicationLogger.Info($"Took {measure.Elapsed} to complete converting to JP index. And got {results.Count()} salary info and {results.Count(r => r.StarterSalary == 0)} results that have salary missing. But {indexes.Values.Count(i => i.SalaryStarter == 0)} indexes missing salary information! from a total of {indexes.Values.Count()}");
return(indexes.Values);
}
private async Task SendNextBatchOfEmailsAsync()
{
var circuitBreaker = await accountsService.GetCircuitBreakerStatusAsync();
if (circuitBreaker.CircuitBreakerStatus != CircuitBreakerStatus.Open)
{
var batchSize = configuration.GetConfig <int>(Constants.BatchSize);
var emailBatch = await accountsService.GetNextBatchOfEmailsAsync(batchSize);
var accountsToProcess = emailBatch.ToList();
applicationLogger.Trace($"About to process email notifications with a batch size of {accountsToProcess.Count}");
var halfOpenCountAllowed = configuration.GetConfig <int>(Constants.GovUkNotifyRetryCount);
foreach (var account in accountsToProcess)
{
try
{
var serviceResponse = await sendCitizenNotificationService.SendCitizenNotificationAsync(account);
if (serviceResponse.RateLimitException)
{
await accountsService.OpenCircuitBreakerAsync();
applicationLogger.Info(
$"RateLimit Exception thrown now resetting the unprocessed email notifications");
await accountsService.SetBatchToCircuitGotBrokenAsync(
accountsToProcess.Where(notification => !notification.Processed));
break;
}
await accountsService.InsertAuditAsync(new AccountNotificationAudit
{
Email = account.EMail,
NotificationProcessingStatus = serviceResponse.Success
? NotificationProcessingStatus.Completed
: NotificationProcessingStatus.Failed
});
if (serviceResponse.Success &&
circuitBreaker.CircuitBreakerStatus == CircuitBreakerStatus.HalfOpen)
{
await accountsService.CloseCircuitBreakerAsync();
}
account.Processed = true;
}
catch (Exception exception)
{
await accountsService.InsertAuditAsync(new AccountNotificationAudit
{
Email = account.EMail,
NotificationProcessingStatus = NotificationProcessingStatus.Failed,
Note = exception.InnerException?.Message
});
await accountsService.HalfOpenCircuitBreakerAsync();
applicationLogger.ErrorJustLogIt("Exception whilst sending email notification", exception);
circuitBreaker = await accountsService.GetCircuitBreakerStatusAsync();
if (circuitBreaker.CircuitBreakerStatus == CircuitBreakerStatus.HalfOpen &&
circuitBreaker.HalfOpenRetryCount == halfOpenCountAllowed)
{
await accountsService.OpenCircuitBreakerAsync();
//Set the all the accountsin the batch that did not get processed (sent ok) to CircuitGotBroken
await accountsService.SetBatchToCircuitGotBrokenAsync(
accountsToProcess.Where(notification => !notification.Processed));
break;
}
}
}
applicationLogger.Trace("Completed processing all accounts in the batch");
}
else
{
applicationLogger.Info("Circuit is open so no account processed");
}
}
public async Task ProcessEmailNotificationsAsyncTests(
CircuitBreakerDetails circuitBreakerDetails,
int batchAccountSize,
SendNotificationResponse sendNotificationResponse,
bool throwSendNotificationException = false,
int halfOpenRetryMax = 5)
{
// Configure Calls
A.CallTo(() => fakeAccountsService.GetCircuitBreakerStatusAsync()).Returns(circuitBreakerDetails);
A.CallTo(() => fakeAccountsService.GetNextBatchOfEmailsAsync(A <int> ._)).Returns(GetAccountsToProcess(batchAccountSize));
if (throwSendNotificationException)
{
A.CallTo(() => fakeSendCitizenNotificationService.SendCitizenNotificationAsync(A <Account> ._)).Throws(() => new Exception(nameof(Exception), new Exception(nameof(Exception))));
}
else
{
A.CallTo(() => fakeSendCitizenNotificationService.SendCitizenNotificationAsync(A <Account> ._)).Returns(sendNotificationResponse);
}
A.CallTo(() => fakeConfiguration.GetConfig <int>(A <string> ._))
.Returns(halfOpenRetryMax);
//For this test the function call is not diasbled in the config.
A.CallTo(() => fakeConfiguration.GetConfig <bool>(A <string> ._)).Returns(false);
// Assign
var emailProcessor = new EmailNotificationProcessor(fakeSendCitizenNotificationService, fakeApplicationLogger, fakeConfiguration, fakeAccountsService);
// Act
await emailProcessor.ProcessEmailNotificationsAsync();
// Assert
A.CallTo(() => fakeAccountsService.GetCircuitBreakerStatusAsync()).MustHaveHappened();
if (circuitBreakerDetails.CircuitBreakerStatus != CircuitBreakerStatus.Open)
{
if (throwSendNotificationException)
{
A.CallTo(() =>
fakeAccountsService.InsertAuditAsync(A <AccountNotificationAudit> .That.Matches(audit =>
audit.NotificationProcessingStatus == NotificationProcessingStatus.Failed && !string.IsNullOrWhiteSpace(audit.Note))))
.MustHaveHappened();
A.CallTo(() => fakeAccountsService.HalfOpenCircuitBreakerAsync()).MustHaveHappened();
if (circuitBreakerDetails.HalfOpenRetryCount == halfOpenRetryMax)
{
A.CallTo(() => fakeAccountsService.SetBatchToCircuitGotBrokenAsync(A <IEnumerable <Account> > ._)).MustHaveHappened();
A.CallTo(() => fakeAccountsService.OpenCircuitBreakerAsync()).MustHaveHappened();
}
}
else
{
if (sendNotificationResponse.Success)
{
A.CallTo(() =>
fakeAccountsService.InsertAuditAsync(A <AccountNotificationAudit> ._)).MustHaveHappened(batchAccountSize, Times.Exactly);
if (circuitBreakerDetails.CircuitBreakerStatus == CircuitBreakerStatus.HalfOpen)
{
A.CallTo(() =>
fakeAccountsService.CloseCircuitBreakerAsync()).MustHaveHappened();
}
}
else
{
if (sendNotificationResponse.RateLimitException)
{
A.CallTo(() => fakeAccountsService.OpenCircuitBreakerAsync()).MustHaveHappened();
A.CallTo(() =>
fakeAccountsService.SetBatchToCircuitGotBrokenAsync(
A <IEnumerable <Account> > ._))
.MustHaveHappened();
A.CallTo(() => fakeApplicationLogger.Info(A <string> ._)).MustHaveHappened();
}
else
{
A.CallTo(() =>
fakeAccountsService.InsertAuditAsync(A <AccountNotificationAudit> .That.Matches(audit =>
audit.NotificationProcessingStatus == NotificationProcessingStatus.Failed)))
.MustHaveHappened();
}
}
}
}
else
{
A.CallTo(() => fakeAccountsService.GetNextBatchOfEmailsAsync(A <int> ._)).MustNotHaveHappened();
A.CallTo(() => fakeSendCitizenNotificationService.SendCitizenNotificationAsync(A <Account> ._)).MustNotHaveHappened();
}
}