public async Task SaveAssembly(BuildProject buildProject)
{
Guard.ArgumentNotNull(buildProject, nameof(buildProject));
if (buildProject.Build != null && !buildProject.Build.Assembly.IsNullOrEmpty())
{
try
{
await _sourceFilesRepositoryPolicy.ExecuteAsync(() => _sourceFilesRepository.SaveAssembly(buildProject.Build.Assembly, buildProject.SpecificationId));
_logger.Information($"Saved assembly for specification id: '{buildProject.SpecificationId}'");
}
catch (Exception ex)
{
string message = $"Failed to save assembly for specification id '{buildProject.SpecificationId}'";
_logger.Error(ex, message);
throw;
}
}
else
{
string message = $"Assembly not present on build project for specification id: '{buildProject.SpecificationId}'";
_logger.Error(message);
throw new ArgumentException(message);
}
}
private async Task <(IEnumerable <ProviderSummary>, long)> GetProviderSummaries(GenerateAllocationMessageProperties messageProperties)
{
_logger.Information($"processing partition index {messageProperties.PartitionIndex} for batch size {messageProperties.PartitionSize}");
int start = messageProperties.PartitionIndex;
int stop = start + messageProperties.PartitionSize - 1;
Stopwatch providerSummaryLookup = Stopwatch.StartNew();
IEnumerable <ProviderSummary> summaries = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.ListRangeAsync <ProviderSummary>(messageProperties.ProviderCacheKey, start, stop));
if (summaries == null)
{
throw new InvalidOperationException("Null provider summaries returned");
}
if (!summaries.Any())
{
throw new InvalidOperationException("No provider summaries returned to process");
}
providerSummaryLookup.Stop();
return(summaries, providerSummaryLookup.ElapsedMilliseconds);
}
private async Task DeletePublishedProviderVersions(string fundingStreamId, string fundingPeriodId)
{
_logger.Information($"Deleting published provider versions for {fundingStreamId} {fundingPeriodId}");
await _publishedFundingRepositoryPolicy.ExecuteAsync(() =>
_publishedFundingRepository.DeleteAllPublishedProviderVersionsByFundingStreamAndPeriod(fundingStreamId, fundingPeriodId));
}
public async Task <IEnumerable <PublishedProvider> > GetPublishedProvidersForApproval(string specificationId, string[] publishedProviderIds = null)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
SpecificationSummary specificationSummary = await _specificationsRepositoryPolicy.ExecuteAsync(
() => _specificationService.GetSpecificationSummaryById(specificationId));
ConcurrentBag <PublishedProvider> results = new ConcurrentBag <PublishedProvider>();
string fundingPeriodId = specificationSummary?.FundingPeriod?.Id;
if (fundingPeriodId.IsNullOrWhitespace())
{
string error = "Could not locate a funding period from the supplied funding period id on the specification summary";
throw new InvalidOperationException(error);
}
List <KeyValuePair <string, string> > allPublishedProviderIds = new List <KeyValuePair <string, string> >();
foreach (Common.Models.Reference fundingStream in specificationSummary.FundingStreams)
{
IEnumerable <KeyValuePair <string, string> > publishedProviders = await _publishedFundingRepositoryPolicy.ExecuteAsync(
() => _publishedFundingRepository.GetPublishedProviderIdsForApproval(fundingStream.Id, fundingPeriodId, publishedProviderIds));
allPublishedProviderIds.AddRange(publishedProviders);
}
return(await _publishedFundingBulkRepository.GetPublishedProviders(allPublishedProviderIds));
}
public TemplateCreateCommandValidator(
IPolicyRepository policyRepository,
IPolicyResiliencePolicies policyResiliencePolicies)
{
Guard.ArgumentNotNull(policyRepository, nameof(policyRepository));
Guard.ArgumentNotNull(policyResiliencePolicies?.PolicyRepository, nameof(policyResiliencePolicies.PolicyRepository));
AsyncPolicy policyRepositoryPolicy = policyResiliencePolicies.PolicyRepository;
RuleFor(x => x.Description).Length(0, 1000);
RuleFor(x => x.FundingStreamId)
.NotEmpty()
.WithMessage("Missing funding stream id")
.MustAsync(async(command, propertyValue, context, cancellationToken) =>
{
FundingStream fundingStream = await policyRepositoryPolicy.ExecuteAsync(() => policyRepository.GetFundingStreamById(command.FundingStreamId));
return(fundingStream != null);
})
.WithMessage("Funding stream id does not exist");
RuleFor(x => x.FundingPeriodId)
.NotEmpty()
.WithMessage("Missing funding period id")
.MustAsync(async(command, propertyValue, context, cancellationToken) =>
{
FundingPeriod fundingPeriod = await policyRepositoryPolicy.ExecuteAsync(() => policyRepository.GetFundingPeriodById(command.FundingPeriodId));
return(fundingPeriod != null);
})
.WithMessage("Funding period id does not exist");
}
public async Task <IEnumerable <PoliciesApiModels.FundingStream> > GetFundingStreams()
{
ApiResponse <IEnumerable <PoliciesApiModels.FundingStream> > fundingStreamsApiResponse
= await _policiesApiClientPolicy.ExecuteAsync(() => _policiesApiClient.GetFundingStreams());
return(fundingStreamsApiResponse.Content);
}
public async Task <IActionResult> SetCurrentProviderVersionForFundingStream(string fundingStreamId,
string providerVersionId, int?providerSnapshotId)
{
ValidationResult validationResult = await _setCurrentRequestValidator.ValidateAsync(new SetFundingStreamCurrentProviderVersionRequest
{
FundingStreamId = fundingStreamId,
ProviderVersionId = providerVersionId
});
if (!validationResult.IsValid)
{
LogError("The set current provider version for funding stream request was invalid." +
$"\n{validationResult.Errors.Select(_ => _.ErrorMessage).Join("\n")}");
return(validationResult.AsBadRequest());
}
await _providerVersionMetadataPolicy.ExecuteAsync(() => _providerVersionMetadata.UpsertCurrentProviderVersion(new CurrentProviderVersion
{
Id = $"Current_{fundingStreamId}",
ProviderVersionId = providerVersionId,
ProviderSnapshotId = providerSnapshotId
}));
return(new NoContentResult());
}
public async Task <IActionResult> GetProfilePatterns(string fundingStreamId,
string fundingPeriodId)
{
Guard.IsNullOrWhiteSpace(fundingStreamId, nameof(fundingStreamId));
Guard.IsNullOrWhiteSpace(fundingPeriodId, nameof(fundingPeriodId));
string cacheKey = CacheKeyForFundingStreamAndFundingPeriod(fundingStreamId, fundingPeriodId);
IEnumerable <FundingStreamPeriodProfilePattern> profilePatterns = await _cacheResilience.ExecuteAsync(() =>
_cacheProvider.GetAsync <FundingStreamPeriodProfilePattern[]>(cacheKey));
if (profilePatterns.IsNullOrEmpty())
{
profilePatterns = await _patternRepositoryResilience.ExecuteAsync(() => _profilePatterns.GetProfilePatternsForFundingStreamAndFundingPeriod(fundingStreamId,
fundingPeriodId));
if (profilePatterns.IsNullOrEmpty())
{
return(new NotFoundResult());
}
await _cacheResilience.ExecuteAsync(() => _cacheProvider.SetAsync(cacheKey, profilePatterns.ToArray(), DateTimeOffset.Now.AddMinutes(30)));
}
return(new OkObjectResult(profilePatterns));
}
/// <summary>
/// Fetches the token based on the username and password
/// </summary>
/// <param name="payload"></param>
/// <returns></returns>
//private Response GetToken(RequestPayload payload)
//{
// var result = new Response();
// try
// {
// var content = JsonConvert.SerializeObject(payload);
// var httpcontent = new StringContent(content, Encoding.UTF8, "application/json");
// using (HttpClient httpClient = new HttpClient())
// {
// httpClient.DefaultRequestHeaders.Accept.Clear();
// httpClient.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json"));
// var response = httpClient.PostAsync("https://ceiusea2-nonprod-core-api.azurewebsites.net/api/Authentication/login", httpcontent).Result;
// if (response.StatusCode == System.Net.HttpStatusCode.OK)
// {
// var responseContent = response.Content.ReadAsStringAsync().Result;
// result = JsonConvert.DeserializeObject<Response>(responseContent);
// }
// }
// }
// catch(Exception ex)
// {
// }
// return result;
//}
/// <summary>
/// Fetches the token based on the username and password
/// </summary>
/// <param name="payload"></param>
/// <returns></returns>
// private async Task<string> GetToken()
// {
// string token = string.Empty;
// var result = new Response();
// var url = "https://test.salesforce.com/services/oauth2/token";
// var postData = new List<KeyValuePair<String, String>>();
// try
// {
// var httpContent = new HttpRequestMessage(HttpMethod.Post, url);
// postData.Add(new KeyValuePair<string, string>("grant_type", "password"));
// postData.Add(new KeyValuePair<string, string>("client_id", "3MVG9_I_oWkIqLrmgPjUWS9SzW7MM9azsg1mg0pzfJHvNrxys1MHQbHHRlUifKtAHkzA4.up5TdGDZF_e4S4U"));
// postData.Add(new KeyValuePair<string, string>("client_secret", "38F7833D193F30499D75A9C17854036A2C9945F4C39628DF5D4B72E5D211E46E"));
// postData.Add(new KeyValuePair<string, string>("username", "[email protected]"));
// postData.Add(new KeyValuePair<string, string>("password", "Harsco123bgZ7CiX2FSJCatQW5f921817a"));
// postData.Add(new KeyValuePair<string, string>("redirect_uri", "https://www.test.salesforce.com/oauth2/callback"));
// httpContent.Content = new FormUrlEncodedContent(postData);
// using (HttpClient httpClient = new HttpClient())
// {
// httpClient.DefaultRequestHeaders.Accept.Clear();
// httpClient.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json"));
// var response =await httpClient.PostAsync("https://test.salesforce.com/services/oauth2/token", httpContent.Content);
// var responseContent = await response.Content.ReadAsStringAsync();
// var responseData = JsonConvert.DeserializeObject<Response>(responseContent);
// if (responseData !=null)
// {
// token = responseData.access_token;
// }
// }
// }
// catch (Exception ex)
// {
// }
// return token;
// }
// private async Task ProcessRequest(string token,string fileToWriteTo)
// {
// using (HttpClient httpClient = new HttpClient())
// {
// httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
// httpClient.DefaultRequestHeaders.Add("Cookie", "BrowserId=Z8rDdaECEeufIO3Mc9aFxg");
// var response = await httpClient.GetAsync("https://cs194.salesforce.com/services/data/v51.0/sobjects/Attachment/00P4X00001ZOldaUAD/Body");
// using (Stream streamToReadFrom = await response.Content.ReadAsStreamAsync())
// {
// // local address
// //string fileToWriteTo = @"D:\Freelance";
// using (Stream streamToWriteTo = System.IO.File.Open(Path.Combine(fileToWriteTo,"test.pdf"), FileMode.Create))
// {
// await streamToReadFrom.CopyToAsync(streamToWriteTo);
// }
// if (System.IO.File.Exists(Path.Combine(fileToWriteTo,"tester.zip")))
// {
// System.IO.File.Delete(Path.Combine(fileToWriteTo, "tester.zip"));
// }
// using (ZipArchive archive = ZipFile.Open(Path.Combine(fileToWriteTo, "tester.zip"), ZipArchiveMode.Create))
// {
// archive.CreateEntryFromFile(Path.Combine(fileToWriteTo, "test.pdf"), "test.pdf");
// }
// if(System.IO.File.Exists(Path.Combine(fileToWriteTo, "test.pdf")))
// {
// System.IO.File.Delete(Path.Combine(fileToWriteTo, "test.pdf"));
// }
// }
// }
// }
//private async Task<string> GetToken()
//{
// string token = string.Empty;
// try
// {
// // get the ids from config
// string EPAUrl = String.Format("https://rcrainfopreprod.epa.gov:443/rcrainfo/rest/api/v1/auth/{0}/{1}", "9fa38bdb-bc22-45f6-9370-babbd98aa5de", "oE1XyRpa8jfMmmDsxEqQ");
// using (HttpClient httpClient = new HttpClient())
// {
// var response = await httpClient.GetAsync(EPAUrl);
// if (response.StatusCode == System.Net.HttpStatusCode.OK)
// {
// var responseContent = await response.Content.ReadAsStringAsync();
// dynamic result = JsonConvert.DeserializeObject<ExpandoObject>(responseContent);
// if (result != null)
// {
// token = result.token;
// }
// }
// }
// }
// catch (Exception ex)
// {
// }
// return token;
//}
//private async Task<string> GetWasteCodes()
//{
// string result = string.Empty;
// try
// {
// string token = await GetToken();
// using (HttpClient httpClient = new HttpClient())
// {
// httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
// var response = await httpClient.GetAsync("https://rcrainfopreprod.epa.gov:443/rcrainfo/rest/api/v1/lookup/federal-waste-codes");
// if (response.StatusCode == System.Net.HttpStatusCode.OK)
// {
// result = await response.Content.ReadAsStringAsync();
// dynamic expandoResult = JsonConvert.DeserializeObject<List<ExpandoObject>>(result);
// }
// }
// }
// catch (Exception ex)
// {
// }
// return result;
//}
private async Task <string> GetWasteCodes()
{
string result = string.Empty;
try
{
string token = await GetTokenRetry("9fa38bdb-bc22-45f6-9370-babbd98aa5de", "oE1XyRpa8jfMmmDsxEqQ");
using (HttpClient httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
await _retryPolicy.ExecuteAsync(async() =>
{
var response = await httpClient.GetAsync("https://rcrainfopreprod.epa.gov:443/rcrainfo/rest/api/v1/lookup/federal-waste-codes");
var responseContent = await response.Content.ReadAsStringAsync();
if (response.StatusCode == System.Net.HttpStatusCode.OK)
{
result = responseContent;
dynamic expandoResult = JsonConvert.DeserializeObject <List <ExpandoObject> >(result);
}
else
{
throw new Exception("Fetching WasteCodes : Received Response code :" + response.StatusCode + " with Message :" + responseContent);
}
});
}
}
catch (Exception ex)
{
}
return(result);
}
public async Task <IActionResult> UpdateBuildProjectRelationships(string specificationId, DatasetRelationshipSummary relationship)
{
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("No specification Id was provided to UpdateBuildProjectRelationships");
return(new BadRequestObjectResult("Null or empty specification Id provided"));
}
if (relationship == null)
{
_logger.Error("A null relationship message was provided to UpdateBuildProjectRelationships");
return(new BadRequestObjectResult("Null relationship provided"));
}
BuildProject buildProject = await GetBuildProjectForSpecificationId(specificationId);
IEnumerable <Models.Calcs.Calculation> calculations = await _calculationsRepository.GetCalculationsBySpecificationId(specificationId);
buildProject.Build = _sourceCodeService.Compile(buildProject, calculations ?? Enumerable.Empty <Models.Calcs.Calculation>());
if (!_featureToggle.IsDynamicBuildProjectEnabled())
{
await _buildProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectsRepository.UpdateBuildProject(buildProject));
}
await _sourceCodeService.SaveAssembly(buildProject);
return(new OkObjectResult(buildProject));
}
private async Task <GherkinParseResult> GetGherkinParseResult(TestScenario testScenario, BuildProject buildProject)
{
Guard.ArgumentNotNull(testScenario, nameof(testScenario));
Guard.ArgumentNotNull(buildProject, nameof(buildProject));
string cacheKey = $"{CacheKeys.GherkinParseResult}{testScenario.Id}";
JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings()
{
TypeNameHandling = TypeNameHandling.All
};
GherkinParseResult gherkinParseResult = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.GetAsync <GherkinParseResult>(cacheKey, jsonSerializerSettings));
if (gherkinParseResult == null)
{
gherkinParseResult = await _parser.Parse(testScenario.SpecificationId, testScenario.Current.Gherkin, buildProject);
if (gherkinParseResult != null && !gherkinParseResult.StepActions.IsNullOrEmpty())
{
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync <GherkinParseResult>(cacheKey, gherkinParseResult, TimeSpan.FromHours(24), true, jsonSerializerSettings));
}
}
return(gherkinParseResult);
}
private async Task <IEnumerable <TukUiAddon> > GetAllAddons(WowClientType clientType)
{
var cacheKey = GetCacheKey(clientType);
var results = await _cacheService.GetCache(cacheKey, async() =>
{
try
{
var query = GetAddonsSuffix(clientType);
var result = await CircuitBreaker.ExecuteAsync(async() =>
await ApiUrl
.SetQueryParam(query, "all")
.WithTimeout(HttpTimeoutSeconds)
.WithHeaders(HttpUtilities.DefaultHeaders)
.GetJsonAsync <List <TukUiAddon> >());
if (clientType.IsRetail())
{
result.Add(await GetTukUiRetailAddon());
result.Add(await GetElvUiRetailAddon());
}
return(result);
}
catch (Exception ex)
{
Log.Error(ex, "Failed to get all addons");
throw;
}
}, 5);
return(results ?? new List <TukUiAddon>());
}
public async Task DeleteFundingSearchDocuments <TIndex>(string fundingStreamId, string fundingPeriodId)
where TIndex : class
{
_logger.Information($"Deleting {typeof(TIndex).Name} documents for {fundingStreamId} {fundingPeriodId}");
ISearchRepository <TIndex> search = GetRepository <TIndex>();
int page = 0;
while (true)
{
SearchResults <TIndex> searchResults = await _searchPolicy
.ExecuteAsync(() => search.Search(string.Empty,
new SearchParameters
{
Top = 50,
SearchMode = SearchMode.Any,
Filter = $"fundingStreamId eq '{fundingStreamId}' and fundingPeriodId eq '{fundingPeriodId}'",
QueryType = QueryType.Full,
Skip = page * 50
}
)
);
if (searchResults == null || searchResults.Results?.Any() == false)
{
return;
}
await _searchPolicy
.ExecuteAsync(() => search.Remove(searchResults.Results.Select(_ => _.Result)));
page += 1;
}
}
/// <summary>
/// Process message once status is received by moving document to the
/// appropriate folder and writing out a log for the processed document
/// </summary>
/// <param name="jobStatusDetail">DataJobStatusDetail object</param>
/// <param name="dataMessage">Name of the file whose status is being processed</param>
/// <param name="cancellationToken">Cancellation token</param>
private async Task PostProcessMessage(DataJobStatusDetail jobStatusDetail, DataMessage dataMessage, CancellationToken cancellationToken)
{
if (jobStatusDetail?.DataJobStatus == null)
{
return;
}
dataMessage.DataJobState = jobStatusDetail.DataJobStatus.DataJobState;
await _retryPolicyForAsyncIo.ExecuteAsync(ct => FileOperationsHelper.WriteStatusFileAsync(dataMessage, _settings.StatusFileExtension, ct), cancellationToken);
switch (dataMessage.DataJobState)
{
case DataJobState.Processed:
{
// Move message file and delete processing status file
_retryPolicyForIo.Execute(() => FileOperationsHelper.MoveDataToTarget(dataMessage.FullPath, Path.Combine(_settings.ProcessingSuccessDir, dataMessage.Name), true));
}
break;
case DataJobState.PostProcessError:
case DataJobState.PreProcessError:
case DataJobState.ProcessedWithErrors:
{
var targetDataMessage = new DataMessage(dataMessage)
{
FullPath = Path.Combine(_settings.ProcessingErrorsDir, dataMessage.Name),
MessageStatus = MessageStatus.Failed
};
_retryPolicyForIo.Execute(() => FileOperationsHelper.MoveDataToTarget(dataMessage.FullPath, targetDataMessage.FullPath));
await _retryPolicyForAsyncIo.ExecuteAsync(ct => FileOperationsHelper.WriteStatusLogFileAsync(jobStatusDetail, targetDataMessage, null, _settings.StatusFileExtension, ct), cancellationToken);
if (_settings.GetExecutionErrors)
{
if (_settings.LogVerbose || Log.IsDebugEnabled)
{
Log.DebugFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_Trying_to_download_execution_errors, _context.JobDetail.Key));
}
var response = await _httpClientHelper.GetExecutionErrors(dataMessage.MessageId, cancellationToken);
if (!response.IsSuccessStatusCode)
{
throw new JobExecutionException(string.Format(Resources.Job_0_download_of_execution_errors_failed_1, _context.JobDetail.Key, string.Format($"Status: {response.StatusCode}. Message: {response.Content}")));
}
using Stream downloadedStream = await response.Content.ReadAsStreamAsync();
var errorsFileName = $"{Path.GetFileNameWithoutExtension(dataMessage.Name)}-ExecutionErrors-{DateTime.Now:yyyy-MM-dd_HH-mm-ss-ffff}.txt";
var errorsFilePath = Path.Combine(Path.GetDirectoryName(dataMessage.FullPath.Replace(_settings.UploadSuccessDir, _settings.ProcessingErrorsDir)), errorsFileName);
var dataMessageForErrorsFile = new DataMessage()
{
FullPath = errorsFilePath,
Name = errorsFileName,
MessageStatus = MessageStatus.Failed
};
await _retryPolicyForAsyncIo.ExecuteAsync(ct => FileOperationsHelper.CreateAsync(downloadedStream, dataMessageForErrorsFile.FullPath, ct), cancellationToken);
}
}
break;
}
}
IEnumerable <Task <SearchResults <ProviderCalculationResultsIndex> > > BuildSearchTasks(SearchModel searchModel, string entityValue, string entityFieldName, string entityExceptionFieldName)
{
IDictionary <string, string> facetDictionary = BuildFacetDictionary(searchModel);
IEnumerable <Task <SearchResults <ProviderCalculationResultsIndex> > > searchTasks = Array.Empty <Task <SearchResults <ProviderCalculationResultsIndex> > >();
if (searchModel.IncludeFacets)
{
foreach (KeyValuePair <string, string> filterPair in facetDictionary)
{
searchTasks = searchTasks.Concat(new[]
{
Task.Run(() =>
{
List <string> searchFields = searchModel.SearchFields?.ToList();
searchFields = searchFields.IsNullOrEmpty() ? new List <string> {
"providerName"
} : searchFields;
return(_searchRepositoryPolicy.ExecuteAsync(() =>
{
return _searchRepository.Search(searchModel.SearchTerm,
new SearchParameters
{
Facets = new[]
{
filterPair.Key
},
SearchMode = (SearchMode)searchModel.SearchMode,
SearchFields = searchFields,
IncludeTotalResultCount = true,
Filter = string.Join(" and ", facetDictionary.Where(x => x.Key != filterPair.Key && !string.IsNullOrWhiteSpace(x.Value)).Select(x => x.Value)),
QueryType = QueryType.Full
});
}));
})
});
}
}
if (facetDictionary.ContainsKey(entityFieldName) && !string.IsNullOrWhiteSpace(entityValue))
{
if (_featureToggle.IsExceptionMessagesEnabled())
{
searchTasks = searchTasks.Concat(new[]
{
BuildCalculationErrorsSearchTask(facetDictionary[entityFieldName], entityValue, entityFieldName, entityExceptionFieldName)
});
}
searchTasks = searchTasks.Concat(new[]
{
BuildCalculationItemsSearchTask(facetDictionary, searchModel, entityValue, entityExceptionFieldName)
});
}
return(searchTasks);
}
public async Task <ApiSpecificationSummary> GetSpecificationSummaryById(string specificationId)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
ApiResponse <ApiSpecificationSummary> specificationSummaryResponse =
await _resiliencePolicy.ExecuteAsync(() => _specifications.GetSpecificationSummaryById(specificationId));
return(specificationSummaryResponse?.Content);
}
private async Task <IEnumerable <IndexError> > IndexDatasetDefinition(DatasetDefinition definition, PoliciesApiModels.FundingStream fundingStream)
{
// Calculate hash for model to see if there are changes
string modelJson = JsonConvert.SerializeObject(definition);
string hashCode = "";
using (SHA256 sha256Generator = SHA256Managed.Create())
{
byte[] modelBytes = UTF8Encoding.UTF8.GetBytes(modelJson);
foreach (byte hashByte in sha256Generator.ComputeHash(modelBytes))
{
hashCode += string.Format("{0:X2}", hashByte);
}
}
DatasetDefinitionIndex datasetDefinitionIndex = new DatasetDefinitionIndex()
{
Id = definition.Id,
Name = definition.Name,
Description = definition.Description,
LastUpdatedDate = DateTimeOffset.Now,
ProviderIdentifier = definition.TableDefinitions.FirstOrDefault()?.FieldDefinitions?.Where(f => f.IdentifierFieldType.HasValue)?.Select(f => Enum.GetName(typeof(IdentifierFieldType), f.IdentifierFieldType.Value)).FirstOrDefault(),
ModelHash = hashCode,
FundingStreamId = fundingStream.Id,
FundingStreamName = fundingStream.Name
};
if (string.IsNullOrWhiteSpace(datasetDefinitionIndex.ProviderIdentifier))
{
datasetDefinitionIndex.ProviderIdentifier = "None";
}
bool updateIndex = true;
// Only update index if metadata or model has changed, this is to preserve the LastUpdateDate
DatasetDefinitionIndex existingIndex = await _datasetDefinitionSearchRepositoryPolicy.ExecuteAsync(() =>
_datasetDefinitionSearchRepository.SearchById(definition.Id));
if (existingIndex != null)
{
if (existingIndex.ModelHash == hashCode &&
existingIndex.Description == definition.Description &&
existingIndex.Name == definition.Name &&
existingIndex.ProviderIdentifier == datasetDefinitionIndex.ProviderIdentifier)
{
updateIndex = false;
}
}
if (updateIndex)
{
return(await _datasetDefinitionSearchRepositoryPolicy.ExecuteAsync(() =>
_datasetDefinitionSearchRepository.Index(new DatasetDefinitionIndex[] { datasetDefinitionIndex })));
}
return(Enumerable.Empty <IndexError>());
}
public async Task <IActionResult> GetFundingStreams()
{
ApiResponse <IEnumerable <PolicyApiClientModel.FundingStream> > fundingStreamsApiResponse
= await _policiesApiClientPolicy.ExecuteAsync(() => _policiesApiClient.GetFundingStreams());
IEnumerable <Models.FundingStream> fundingStreams =
_mapper.Map <IEnumerable <Models.FundingStream> >(fundingStreamsApiResponse.Content);
return(new OkObjectResult(fundingStreams));
}
public Task RegisterBusHandlerAsync(IRequestMatcher pattern, Func <Message, CancellationToken, Task> handler, RouteExecution execution = RouteExecution.Asynchronous, RouteMode mode = RouteMode.Observe, CancellationToken cancellationToken = default)
{
return(_policy.ExecuteAsync((_, registerPolicyToken) => _registration.RegisterBusHandlerAsync(
pattern,
(message, handlerToken) => _policy.ExecuteAsync((__, handlerPolicyToken) => handler(message, handlerPolicyToken), new Context(), handlerToken),
execution,
mode,
registerPolicyToken
), new Context(), cancellationToken));
}
private async Task CacheJob(Job job)
{
// don't cache the job if it's been superseded or there is no associated specification
if (job.CompletionStatus == CompletionStatus.Superseded || string.IsNullOrWhiteSpace(job.SpecificationId))
{
return;
}
string cacheKey = $"{CacheKeys.LatestJobs}{job.SpecificationId}:{job.JobDefinitionId}";
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync(cacheKey, job));
}
private async Task <TEntity> QuerySingle <TEntity>(string sql,
CommandType commandType,
object parameters)
{
using IDbConnection connection = NewOpenConnection();
return(await _queryAsyncPolicy.ExecuteAsync(() => connection.QuerySingleOrDefaultAsync <TEntity>(sql,
parameters ?? new
{
},
commandType: commandType)));
}
private static async Task DownloadItem(KeyValuePair<string, string> item, Uri uri)
{
try
{
Logger.Log($"Starting download for {uri}").FireAndForget();
bool isSmall = false;
using (var client = new HttpClient())
{
HttpRequestMessage m = new HttpRequestMessage(HttpMethod.Head, uri);
HttpResponseMessage resp = await client.SendAsync(m);
if (resp.Content.Headers.ContentLength < 100) { isSmall = true; }
if (resp.Content.Headers.ContentType != null && (resp.Content.Headers.ContentType.ToString() == "text/plain" || resp.Content.Headers.ContentType.ToString() == "text/json"))
{
isSmall = true;
}
m.Dispose();
}
Directory.CreateDirectory(Path.GetDirectoryName(NormalizePath(item.Key)));
try
{
if (isSmall)
{
DownloadService ds = new DownloadService(downloadOptSmall);
await RetryPolicy.ExecuteAsync(async () => { await ds.DownloadFileAsync(uri.ToString(), item.Key); });
}
else
{
await RetryPolicy.ExecuteAsync(async () => { await new DownloadService(downloadOpt).DownloadFileAsync(uri.ToString(), item.Key); });
}
}
catch (System.IO.InvalidDataException)
{
await RetryPolicy.ExecuteAsync(async () => { await DownloadManually(uri, item.Key); });
}
Logger.Log($"Download for {uri} finished.").FireAndForget();
}
catch (System.Net.WebException e)
{
if (e.Message == "The remote server returned an error: (404) Not Found.")
{
Logger.Log("Could not download " + uri + ": 404 not found.", LogType.ERROR).FireAndForget();
}
else { throw; }
}
}
private async Task <(Stream assembly, string etag)> GetAssemblyFromBlobStorage(string specificationId)
{
ICloudBlob assemblyBlob = await GetAssemblyCloudBlob(specificationId);
if (assemblyBlob == null)
{
return(null, null);
}
Stream assembly = await _blobResilience.ExecuteAsync(() => _blobs.DownloadToStreamAsync(assemblyBlob));
return(assembly, assemblyBlob.Properties.ETag);
}
/// <summary>
/// Processes download queue by downloading each
/// file to download success location.
/// </summary>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>
/// Task object for continuation
/// </returns>
/// <exception cref="System.Exception"></exception>
private async Task ProcessDownloadQueue(CancellationToken cancellationToken)
{
var fileCount = 0;
while (DownloadQueue.TryDequeue(out DataMessage dataMessage))
{
var response = await _httpClientHelper.GetRequestAsync(new UriBuilder(dataMessage.DownloadLocation).Uri, true, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
if (!response.IsSuccessStatusCode)
{
throw new JobExecutionException(string.Format(Resources.Job_0_Download_failure_1, _context.JobDetail.Key, response.StatusCode));
}
using (var downloadedStream = await response.Content.ReadAsStreamAsync())
{
if (fileCount > 0 && _settings.DelayBetweenFiles > 0) //Only delay after first file and never after last.
{
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(_settings.DelayBetweenFiles));
}
fileCount++;
cancellationToken.ThrowIfCancellationRequested();
//Downloaded file has no file name. We need to create it.
//It will be timestamp followed by number in this download batch.
var fileName = $"{DateTime.Now:yyyy-MM-dd_HH-mm-ss-ffff}-{fileCount:D6}";
fileName = Path.ChangeExtension(fileName, "zip");
dataMessage.FullPath = Path.Combine(_settings.DownloadSuccessDir, fileName);
dataMessage.Name = fileName;
dataMessage.MessageStatus = MessageStatus.Succeeded;
await _retryPolicyForAsyncIo.ExecuteAsync(ct => FileOperationsHelper.CreateAsync(downloadedStream, dataMessage.FullPath, ct), cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
}
if (_settings.LogVerbose || Log.IsDebugEnabled)
{
Log.DebugFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_File_1_was_downloaded, _context.JobDetail.Key, dataMessage.FullPath.Replace(@"{", @"{{").Replace(@"}", @"}}")));
}
await AcknowledgeDownload(dataMessage, cancellationToken);
if (_settings.UnzipPackage)
{
await _retryPolicyForAsyncIo.ExecuteAsync(ct => FileOperationsHelper.UnzipPackageAsync(dataMessage.FullPath, _settings.DeletePackage, _settings.AddTimestamp, ct), cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
}
}
}
public async Task <IActionResult> AssignProfilePatternKey(
string fundingStreamId,
string fundingPeriodId,
string providerId,
ProfilePatternKey profilePatternKey,
Reference author)
{
Guard.IsNullOrWhiteSpace(fundingStreamId, nameof(fundingStreamId));
Guard.IsNullOrWhiteSpace(fundingPeriodId, nameof(fundingPeriodId));
Guard.IsNullOrWhiteSpace(providerId, nameof(providerId));
Guard.ArgumentNotNull(profilePatternKey, nameof(profilePatternKey));
PublishedProvider publishedProvider = await _publishingResiliencePolicy.ExecuteAsync(async() =>
await _publishedFundingRepository.GetPublishedProvider(fundingStreamId, fundingPeriodId, providerId));
if (publishedProvider == null)
{
return(new StatusCodeResult((int)HttpStatusCode.NotFound));
}
FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStreamId, fundingPeriodId);
if (fundingConfiguration == null || !fundingConfiguration.EnableUserEditableRuleBasedProfiles)
{
return(new BadRequestObjectResult($"User not allowed to edit rule based profiles for funding stream - '{fundingStreamId}' and funding period - '{fundingPeriodId}'"));
}
if (MatchingProfilePatternKeyExists(publishedProvider.Current, profilePatternKey))
{
return(new StatusCodeResult((int)HttpStatusCode.NotModified));
}
PublishedProvider modifiedPublishedProvider = await CreateVersion(publishedProvider, author);
if (modifiedPublishedProvider == null)
{
return(new StatusCodeResult((int)HttpStatusCode.BadRequest));
}
PublishedProviderVersion newPublishedProviderVersion = publishedProvider.Current;
newPublishedProviderVersion.SetProfilePatternKey(profilePatternKey, author);
await ProfileFundingLineValues(newPublishedProviderVersion, profilePatternKey);
await _publishedProviderErrorDetection.ProcessPublishedProvider(publishedProvider, _ => _ is FundingLineValueProfileMismatchErrorDetector);
await SavePublishedProvider(publishedProvider, newPublishedProviderVersion);
return(new StatusCodeResult((int)HttpStatusCode.OK));
}
private async Task RemoveBlob(string name, string container)
{
try
{
ICloudBlob blob = await _resilience.ExecuteAsync(() =>
_blobClient.GetBlobReferenceFromServerAsync(name, container));
await _resilience.ExecuteAsync(() => blob.DeleteIfExistsAsync());
}
catch (Exception e)
{
_logger.Error(e, $"Unable to delete blob {name} from {container}");
}
}
public void InterceptSynchronous(IInvocation invocation)
{
var capture = invocation.CaptureProceedInfo();
invocation.ReturnValue = _asyncPolicy
.ExecuteAsync(() =>
{
capture.Invoke();
return(Task.FromResult(invocation.ReturnValue));
})
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
}
public async Task <Container> InitializeCollection(CosmosClient client)
{
Database database = client.GetDatabase(_cosmosDataStoreConfiguration.DatabaseId);
Container containerClient = database.GetContainer(_cosmosCollectionConfiguration.CollectionId);
_logger.LogInformation("Finding Container: {collectionId}", _cosmosCollectionConfiguration.CollectionId);
AsyncPolicy retryPolicy = _retryExceptionPolicyFactory.GetRetryPolicy();
var existingContainer = await retryPolicy.ExecuteAsync(async() => await database.TryGetContainerAsync(_cosmosCollectionConfiguration.CollectionId));
_logger.LogInformation("Creating Cosmos Container if not exits: {collectionId}", _cosmosCollectionConfiguration.CollectionId);
ContainerResponse containerResponse = await retryPolicy.ExecuteAsync(async() =>
await database.CreateContainerIfNotExistsAsync(
_cosmosCollectionConfiguration.CollectionId,
$"/{KnownDocumentProperties.PartitionKey}",
_cosmosCollectionConfiguration.InitialCollectionThroughput));
if (containerResponse.StatusCode == HttpStatusCode.Created || containerResponse.Resource.DefaultTimeToLive != -1)
{
if (_cosmosCollectionConfiguration.InitialCollectionThroughput.HasValue)
{
var throughputProperties = ThroughputProperties.CreateManualThroughput(_cosmosCollectionConfiguration.InitialCollectionThroughput.Value);
await retryPolicy.ExecuteAsync(async() => await containerClient.ReplaceThroughputAsync(throughputProperties));
}
containerResponse.Resource.DefaultTimeToLive = -1;
existingContainer = await retryPolicy.ExecuteAsync(async() => await containerClient.ReplaceContainerAsync(containerResponse));
}
await retryPolicy.ExecuteAsync(async() =>
{
try
{
await _clientTestProvider.PerformTest(existingContainer, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration);
}
catch (CosmosException e) when(e.StatusCode == HttpStatusCode.TooManyRequests)
{
// This is the very first interaction with the collection, and we might get this exception
// when it calls GetCachedContainerPropertiesAsync, which does not use our request handler.
throw new RequestRateExceededException(e.RetryAfter);
}
});
await _upgradeManager.SetupContainerAsync(containerClient);
return(existingContainer);
}
private async Task <IEnumerable <FundingConfiguration> > GetFundingConfigurationsByFundingStreamId(string fundingStreamId)
{
ApiResponse <IEnumerable <FundingConfiguration> > fundingConfigResponse
= await _policiesApiClientPolicy.ExecuteAsync(() =>
_policiesApiClient.GetFundingConfigurationsByFundingStreamId(fundingStreamId));
if (!fundingConfigResponse.StatusCode.IsSuccess())
{
string errorMessage = $"Unable to retrieve funding configs for funding stream with ID: {fundingStreamId}";
_logger.Error(errorMessage);
return(Enumerable.Empty <FundingConfiguration>());
}
return(fundingConfigResponse.Content);
}
/// <inheritdoc />
public async Task AddOrReplaceFileMetaToTransactionAsync(
FileMeta fileMeta,
string transactionId,
string fileId,
CancellationToken cancellationToken = default)
{
await retryPolicy
.ExecuteAsync(
ct => client.AddOrReplaceFileMetaToTransactionAsync(
fileMeta,
transactionId,
fileId,
ct),
cancellationToken);
}
