public async Task SaveAssembly(BuildProject buildProject) { Guard.ArgumentNotNull(buildProject, nameof(buildProject)); if (buildProject.Build != null && !buildProject.Build.Assembly.IsNullOrEmpty()) { try { await _sourceFilesRepositoryPolicy.ExecuteAsync(() => _sourceFilesRepository.SaveAssembly(buildProject.Build.Assembly, buildProject.SpecificationId)); _logger.Information($"Saved assembly for specification id: '{buildProject.SpecificationId}'"); } catch (Exception ex) { string message = $"Failed to save assembly for specification id '{buildProject.SpecificationId}'"; _logger.Error(ex, message); throw; } } else { string message = $"Assembly not present on build project for specification id: '{buildProject.SpecificationId}'"; _logger.Error(message); throw new ArgumentException(message); } }
private async Task <(IEnumerable <ProviderSummary>, long)> GetProviderSummaries(GenerateAllocationMessageProperties messageProperties) { _logger.Information($"processing partition index {messageProperties.PartitionIndex} for batch size {messageProperties.PartitionSize}"); int start = messageProperties.PartitionIndex; int stop = start + messageProperties.PartitionSize - 1; Stopwatch providerSummaryLookup = Stopwatch.StartNew(); IEnumerable <ProviderSummary> summaries = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.ListRangeAsync <ProviderSummary>(messageProperties.ProviderCacheKey, start, stop)); if (summaries == null) { throw new InvalidOperationException("Null provider summaries returned"); } if (!summaries.Any()) { throw new InvalidOperationException("No provider summaries returned to process"); } providerSummaryLookup.Stop(); return(summaries, providerSummaryLookup.ElapsedMilliseconds); }
private async Task DeletePublishedProviderVersions(string fundingStreamId, string fundingPeriodId) { _logger.Information($"Deleting published provider versions for {fundingStreamId} {fundingPeriodId}"); await _publishedFundingRepositoryPolicy.ExecuteAsync(() => _publishedFundingRepository.DeleteAllPublishedProviderVersionsByFundingStreamAndPeriod(fundingStreamId, fundingPeriodId)); }
public async Task <IEnumerable <PublishedProvider> > GetPublishedProvidersForApproval(string specificationId, string[] publishedProviderIds = null) { Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId)); SpecificationSummary specificationSummary = await _specificationsRepositoryPolicy.ExecuteAsync( () => _specificationService.GetSpecificationSummaryById(specificationId)); ConcurrentBag <PublishedProvider> results = new ConcurrentBag <PublishedProvider>(); string fundingPeriodId = specificationSummary?.FundingPeriod?.Id; if (fundingPeriodId.IsNullOrWhitespace()) { string error = "Could not locate a funding period from the supplied funding period id on the specification summary"; throw new InvalidOperationException(error); } List <KeyValuePair <string, string> > allPublishedProviderIds = new List <KeyValuePair <string, string> >(); foreach (Common.Models.Reference fundingStream in specificationSummary.FundingStreams) { IEnumerable <KeyValuePair <string, string> > publishedProviders = await _publishedFundingRepositoryPolicy.ExecuteAsync( () => _publishedFundingRepository.GetPublishedProviderIdsForApproval(fundingStream.Id, fundingPeriodId, publishedProviderIds)); allPublishedProviderIds.AddRange(publishedProviders); } return(await _publishedFundingBulkRepository.GetPublishedProviders(allPublishedProviderIds)); }
public TemplateCreateCommandValidator( IPolicyRepository policyRepository, IPolicyResiliencePolicies policyResiliencePolicies) { Guard.ArgumentNotNull(policyRepository, nameof(policyRepository)); Guard.ArgumentNotNull(policyResiliencePolicies?.PolicyRepository, nameof(policyResiliencePolicies.PolicyRepository)); AsyncPolicy policyRepositoryPolicy = policyResiliencePolicies.PolicyRepository; RuleFor(x => x.Description).Length(0, 1000); RuleFor(x => x.FundingStreamId) .NotEmpty() .WithMessage("Missing funding stream id") .MustAsync(async(command, propertyValue, context, cancellationToken) => { FundingStream fundingStream = await policyRepositoryPolicy.ExecuteAsync(() => policyRepository.GetFundingStreamById(command.FundingStreamId)); return(fundingStream != null); }) .WithMessage("Funding stream id does not exist"); RuleFor(x => x.FundingPeriodId) .NotEmpty() .WithMessage("Missing funding period id") .MustAsync(async(command, propertyValue, context, cancellationToken) => { FundingPeriod fundingPeriod = await policyRepositoryPolicy.ExecuteAsync(() => policyRepository.GetFundingPeriodById(command.FundingPeriodId)); return(fundingPeriod != null); }) .WithMessage("Funding period id does not exist"); }
public async Task <IEnumerable <PoliciesApiModels.FundingStream> > GetFundingStreams() { ApiResponse <IEnumerable <PoliciesApiModels.FundingStream> > fundingStreamsApiResponse = await _policiesApiClientPolicy.ExecuteAsync(() => _policiesApiClient.GetFundingStreams()); return(fundingStreamsApiResponse.Content); }
public async Task <IActionResult> SetCurrentProviderVersionForFundingStream(string fundingStreamId, string providerVersionId, int?providerSnapshotId) { ValidationResult validationResult = await _setCurrentRequestValidator.ValidateAsync(new SetFundingStreamCurrentProviderVersionRequest { FundingStreamId = fundingStreamId, ProviderVersionId = providerVersionId }); if (!validationResult.IsValid) { LogError("The set current provider version for funding stream request was invalid." + $"\n{validationResult.Errors.Select(_ => _.ErrorMessage).Join("\n")}"); return(validationResult.AsBadRequest()); } await _providerVersionMetadataPolicy.ExecuteAsync(() => _providerVersionMetadata.UpsertCurrentProviderVersion(new CurrentProviderVersion { Id = $"Current_{fundingStreamId}", ProviderVersionId = providerVersionId, ProviderSnapshotId = providerSnapshotId })); return(new NoContentResult()); }
public async Task <IActionResult> GetProfilePatterns(string fundingStreamId, string fundingPeriodId) { Guard.IsNullOrWhiteSpace(fundingStreamId, nameof(fundingStreamId)); Guard.IsNullOrWhiteSpace(fundingPeriodId, nameof(fundingPeriodId)); string cacheKey = CacheKeyForFundingStreamAndFundingPeriod(fundingStreamId, fundingPeriodId); IEnumerable <FundingStreamPeriodProfilePattern> profilePatterns = await _cacheResilience.ExecuteAsync(() => _cacheProvider.GetAsync <FundingStreamPeriodProfilePattern[]>(cacheKey)); if (profilePatterns.IsNullOrEmpty()) { profilePatterns = await _patternRepositoryResilience.ExecuteAsync(() => _profilePatterns.GetProfilePatternsForFundingStreamAndFundingPeriod(fundingStreamId, fundingPeriodId)); if (profilePatterns.IsNullOrEmpty()) { return(new NotFoundResult()); } await _cacheResilience.ExecuteAsync(() => _cacheProvider.SetAsync(cacheKey, profilePatterns.ToArray(), DateTimeOffset.Now.AddMinutes(30))); } return(new OkObjectResult(profilePatterns)); }
/// <summary> /// Fetches the token based on the username and password /// </summary> /// <param name="payload"></param> /// <returns></returns> //private Response GetToken(RequestPayload payload) //{ // var result = new Response(); // try // { // var content = JsonConvert.SerializeObject(payload); // var httpcontent = new StringContent(content, Encoding.UTF8, "application/json"); // using (HttpClient httpClient = new HttpClient()) // { // httpClient.DefaultRequestHeaders.Accept.Clear(); // httpClient.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json")); // var response = httpClient.PostAsync("https://ceiusea2-nonprod-core-api.azurewebsites.net/api/Authentication/login", httpcontent).Result; // if (response.StatusCode == System.Net.HttpStatusCode.OK) // { // var responseContent = response.Content.ReadAsStringAsync().Result; // result = JsonConvert.DeserializeObject<Response>(responseContent); // } // } // } // catch(Exception ex) // { // } // return result; //} /// <summary> /// Fetches the token based on the username and password /// </summary> /// <param name="payload"></param> /// <returns></returns> // private async Task<string> GetToken() // { // string token = string.Empty; // var result = new Response(); // var url = "https://test.salesforce.com/services/oauth2/token"; // var postData = new List<KeyValuePair<String, String>>(); // try // { // var httpContent = new HttpRequestMessage(HttpMethod.Post, url); // postData.Add(new KeyValuePair<string, string>("grant_type", "password")); // postData.Add(new KeyValuePair<string, string>("client_id", "3MVG9_I_oWkIqLrmgPjUWS9SzW7MM9azsg1mg0pzfJHvNrxys1MHQbHHRlUifKtAHkzA4.up5TdGDZF_e4S4U")); // postData.Add(new KeyValuePair<string, string>("client_secret", "38F7833D193F30499D75A9C17854036A2C9945F4C39628DF5D4B72E5D211E46E")); // postData.Add(new KeyValuePair<string, string>("username", "[email protected]")); // postData.Add(new KeyValuePair<string, string>("password", "Harsco123bgZ7CiX2FSJCatQW5f921817a")); // postData.Add(new KeyValuePair<string, string>("redirect_uri", "https://www.test.salesforce.com/oauth2/callback")); // httpContent.Content = new FormUrlEncodedContent(postData); // using (HttpClient httpClient = new HttpClient()) // { // httpClient.DefaultRequestHeaders.Accept.Clear(); // httpClient.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json")); // var response =await httpClient.PostAsync("https://test.salesforce.com/services/oauth2/token", httpContent.Content); // var responseContent = await response.Content.ReadAsStringAsync(); // var responseData = JsonConvert.DeserializeObject<Response>(responseContent); // if (responseData !=null) // { // token = responseData.access_token; // } // } // } // catch (Exception ex) // { // } // return token; // } // private async Task ProcessRequest(string token,string fileToWriteTo) // { // using (HttpClient httpClient = new HttpClient()) // { // httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); // httpClient.DefaultRequestHeaders.Add("Cookie", "BrowserId=Z8rDdaECEeufIO3Mc9aFxg"); // var response = await httpClient.GetAsync("https://cs194.salesforce.com/services/data/v51.0/sobjects/Attachment/00P4X00001ZOldaUAD/Body"); // using (Stream streamToReadFrom = await response.Content.ReadAsStreamAsync()) // { // // local address // //string fileToWriteTo = @"D:\Freelance"; // using (Stream streamToWriteTo = System.IO.File.Open(Path.Combine(fileToWriteTo,"test.pdf"), FileMode.Create)) // { // await streamToReadFrom.CopyToAsync(streamToWriteTo); // } // if (System.IO.File.Exists(Path.Combine(fileToWriteTo,"tester.zip"))) // { // System.IO.File.Delete(Path.Combine(fileToWriteTo, "tester.zip")); // } // using (ZipArchive archive = ZipFile.Open(Path.Combine(fileToWriteTo, "tester.zip"), ZipArchiveMode.Create)) // { // archive.CreateEntryFromFile(Path.Combine(fileToWriteTo, "test.pdf"), "test.pdf"); // } // if(System.IO.File.Exists(Path.Combine(fileToWriteTo, "test.pdf"))) // { // System.IO.File.Delete(Path.Combine(fileToWriteTo, "test.pdf")); // } // } // } // } //private async Task<string> GetToken() //{ // string token = string.Empty; // try // { // // get the ids from config // string EPAUrl = String.Format("https://rcrainfopreprod.epa.gov:443/rcrainfo/rest/api/v1/auth/{0}/{1}", "9fa38bdb-bc22-45f6-9370-babbd98aa5de", "oE1XyRpa8jfMmmDsxEqQ"); // using (HttpClient httpClient = new HttpClient()) // { // var response = await httpClient.GetAsync(EPAUrl); // if (response.StatusCode == System.Net.HttpStatusCode.OK) // { // var responseContent = await response.Content.ReadAsStringAsync(); // dynamic result = JsonConvert.DeserializeObject<ExpandoObject>(responseContent); // if (result != null) // { // token = result.token; // } // } // } // } // catch (Exception ex) // { // } // return token; //} //private async Task<string> GetWasteCodes() //{ // string result = string.Empty; // try // { // string token = await GetToken(); // using (HttpClient httpClient = new HttpClient()) // { // httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); // var response = await httpClient.GetAsync("https://rcrainfopreprod.epa.gov:443/rcrainfo/rest/api/v1/lookup/federal-waste-codes"); // if (response.StatusCode == System.Net.HttpStatusCode.OK) // { // result = await response.Content.ReadAsStringAsync(); // dynamic expandoResult = JsonConvert.DeserializeObject<List<ExpandoObject>>(result); // } // } // } // catch (Exception ex) // { // } // return result; //} private async Task <string> GetWasteCodes() { string result = string.Empty; try { string token = await GetTokenRetry("9fa38bdb-bc22-45f6-9370-babbd98aa5de", "oE1XyRpa8jfMmmDsxEqQ"); using (HttpClient httpClient = new HttpClient()) { httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); await _retryPolicy.ExecuteAsync(async() => { var response = await httpClient.GetAsync("https://rcrainfopreprod.epa.gov:443/rcrainfo/rest/api/v1/lookup/federal-waste-codes"); var responseContent = await response.Content.ReadAsStringAsync(); if (response.StatusCode == System.Net.HttpStatusCode.OK) { result = responseContent; dynamic expandoResult = JsonConvert.DeserializeObject <List <ExpandoObject> >(result); } else { throw new Exception("Fetching WasteCodes : Received Response code :" + response.StatusCode + " with Message :" + responseContent); } }); } } catch (Exception ex) { } return(result); }
public async Task <IActionResult> UpdateBuildProjectRelationships(string specificationId, DatasetRelationshipSummary relationship) { if (string.IsNullOrWhiteSpace(specificationId)) { _logger.Error("No specification Id was provided to UpdateBuildProjectRelationships"); return(new BadRequestObjectResult("Null or empty specification Id provided")); } if (relationship == null) { _logger.Error("A null relationship message was provided to UpdateBuildProjectRelationships"); return(new BadRequestObjectResult("Null relationship provided")); } BuildProject buildProject = await GetBuildProjectForSpecificationId(specificationId); IEnumerable <Models.Calcs.Calculation> calculations = await _calculationsRepository.GetCalculationsBySpecificationId(specificationId); buildProject.Build = _sourceCodeService.Compile(buildProject, calculations ?? Enumerable.Empty <Models.Calcs.Calculation>()); if (!_featureToggle.IsDynamicBuildProjectEnabled()) { await _buildProjectsRepositoryPolicy.ExecuteAsync(() => _buildProjectsRepository.UpdateBuildProject(buildProject)); } await _sourceCodeService.SaveAssembly(buildProject); return(new OkObjectResult(buildProject)); }
private async Task <GherkinParseResult> GetGherkinParseResult(TestScenario testScenario, BuildProject buildProject) { Guard.ArgumentNotNull(testScenario, nameof(testScenario)); Guard.ArgumentNotNull(buildProject, nameof(buildProject)); string cacheKey = $"{CacheKeys.GherkinParseResult}{testScenario.Id}"; JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.All }; GherkinParseResult gherkinParseResult = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.GetAsync <GherkinParseResult>(cacheKey, jsonSerializerSettings)); if (gherkinParseResult == null) { gherkinParseResult = await _parser.Parse(testScenario.SpecificationId, testScenario.Current.Gherkin, buildProject); if (gherkinParseResult != null && !gherkinParseResult.StepActions.IsNullOrEmpty()) { await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync <GherkinParseResult>(cacheKey, gherkinParseResult, TimeSpan.FromHours(24), true, jsonSerializerSettings)); } } return(gherkinParseResult); }
private async Task <IEnumerable <TukUiAddon> > GetAllAddons(WowClientType clientType) { var cacheKey = GetCacheKey(clientType); var results = await _cacheService.GetCache(cacheKey, async() => { try { var query = GetAddonsSuffix(clientType); var result = await CircuitBreaker.ExecuteAsync(async() => await ApiUrl .SetQueryParam(query, "all") .WithTimeout(HttpTimeoutSeconds) .WithHeaders(HttpUtilities.DefaultHeaders) .GetJsonAsync <List <TukUiAddon> >()); if (clientType.IsRetail()) { result.Add(await GetTukUiRetailAddon()); result.Add(await GetElvUiRetailAddon()); } return(result); } catch (Exception ex) { Log.Error(ex, "Failed to get all addons"); throw; } }, 5); return(results ?? new List <TukUiAddon>()); }
public async Task DeleteFundingSearchDocuments <TIndex>(string fundingStreamId, string fundingPeriodId) where TIndex : class { _logger.Information($"Deleting {typeof(TIndex).Name} documents for {fundingStreamId} {fundingPeriodId}"); ISearchRepository <TIndex> search = GetRepository <TIndex>(); int page = 0; while (true) { SearchResults <TIndex> searchResults = await _searchPolicy .ExecuteAsync(() => search.Search(string.Empty, new SearchParameters { Top = 50, SearchMode = SearchMode.Any, Filter = $"fundingStreamId eq '{fundingStreamId}' and fundingPeriodId eq '{fundingPeriodId}'", QueryType = QueryType.Full, Skip = page * 50 } ) ); if (searchResults == null || searchResults.Results?.Any() == false) { return; } await _searchPolicy .ExecuteAsync(() => search.Remove(searchResults.Results.Select(_ => _.Result))); page += 1; } }
/// <summary> /// Process message once status is received by moving document to the /// appropriate folder and writing out a log for the processed document /// </summary> /// <param name="jobStatusDetail">DataJobStatusDetail object</param> /// <param name="dataMessage">Name of the file whose status is being processed</param> /// <param name="cancellationToken">Cancellation token</param> private async Task PostProcessMessage(DataJobStatusDetail jobStatusDetail, DataMessage dataMessage, CancellationToken cancellationToken) { if (jobStatusDetail?.DataJobStatus == null) { return; } dataMessage.DataJobState = jobStatusDetail.DataJobStatus.DataJobState; await _retryPolicyForAsyncIo.ExecuteAsync(ct => FileOperationsHelper.WriteStatusFileAsync(dataMessage, _settings.StatusFileExtension, ct), cancellationToken); switch (dataMessage.DataJobState) { case DataJobState.Processed: { // Move message file and delete processing status file _retryPolicyForIo.Execute(() => FileOperationsHelper.MoveDataToTarget(dataMessage.FullPath, Path.Combine(_settings.ProcessingSuccessDir, dataMessage.Name), true)); } break; case DataJobState.PostProcessError: case DataJobState.PreProcessError: case DataJobState.ProcessedWithErrors: { var targetDataMessage = new DataMessage(dataMessage) { FullPath = Path.Combine(_settings.ProcessingErrorsDir, dataMessage.Name), MessageStatus = MessageStatus.Failed }; _retryPolicyForIo.Execute(() => FileOperationsHelper.MoveDataToTarget(dataMessage.FullPath, targetDataMessage.FullPath)); await _retryPolicyForAsyncIo.ExecuteAsync(ct => FileOperationsHelper.WriteStatusLogFileAsync(jobStatusDetail, targetDataMessage, null, _settings.StatusFileExtension, ct), cancellationToken); if (_settings.GetExecutionErrors) { if (_settings.LogVerbose || Log.IsDebugEnabled) { Log.DebugFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_Trying_to_download_execution_errors, _context.JobDetail.Key)); } var response = await _httpClientHelper.GetExecutionErrors(dataMessage.MessageId, cancellationToken); if (!response.IsSuccessStatusCode) { throw new JobExecutionException(string.Format(Resources.Job_0_download_of_execution_errors_failed_1, _context.JobDetail.Key, string.Format($"Status: {response.StatusCode}. Message: {response.Content}"))); } using Stream downloadedStream = await response.Content.ReadAsStreamAsync(); var errorsFileName = $"{Path.GetFileNameWithoutExtension(dataMessage.Name)}-ExecutionErrors-{DateTime.Now:yyyy-MM-dd_HH-mm-ss-ffff}.txt"; var errorsFilePath = Path.Combine(Path.GetDirectoryName(dataMessage.FullPath.Replace(_settings.UploadSuccessDir, _settings.ProcessingErrorsDir)), errorsFileName); var dataMessageForErrorsFile = new DataMessage() { FullPath = errorsFilePath, Name = errorsFileName, MessageStatus = MessageStatus.Failed }; await _retryPolicyForAsyncIo.ExecuteAsync(ct => FileOperationsHelper.CreateAsync(downloadedStream, dataMessageForErrorsFile.FullPath, ct), cancellationToken); } } break; } }
IEnumerable <Task <SearchResults <ProviderCalculationResultsIndex> > > BuildSearchTasks(SearchModel searchModel, string entityValue, string entityFieldName, string entityExceptionFieldName) { IDictionary <string, string> facetDictionary = BuildFacetDictionary(searchModel); IEnumerable <Task <SearchResults <ProviderCalculationResultsIndex> > > searchTasks = Array.Empty <Task <SearchResults <ProviderCalculationResultsIndex> > >(); if (searchModel.IncludeFacets) { foreach (KeyValuePair <string, string> filterPair in facetDictionary) { searchTasks = searchTasks.Concat(new[] { Task.Run(() => { List <string> searchFields = searchModel.SearchFields?.ToList(); searchFields = searchFields.IsNullOrEmpty() ? new List <string> { "providerName" } : searchFields; return(_searchRepositoryPolicy.ExecuteAsync(() => { return _searchRepository.Search(searchModel.SearchTerm, new SearchParameters { Facets = new[] { filterPair.Key }, SearchMode = (SearchMode)searchModel.SearchMode, SearchFields = searchFields, IncludeTotalResultCount = true, Filter = string.Join(" and ", facetDictionary.Where(x => x.Key != filterPair.Key && !string.IsNullOrWhiteSpace(x.Value)).Select(x => x.Value)), QueryType = QueryType.Full }); })); }) }); } } if (facetDictionary.ContainsKey(entityFieldName) && !string.IsNullOrWhiteSpace(entityValue)) { if (_featureToggle.IsExceptionMessagesEnabled()) { searchTasks = searchTasks.Concat(new[] { BuildCalculationErrorsSearchTask(facetDictionary[entityFieldName], entityValue, entityFieldName, entityExceptionFieldName) }); } searchTasks = searchTasks.Concat(new[] { BuildCalculationItemsSearchTask(facetDictionary, searchModel, entityValue, entityExceptionFieldName) }); } return(searchTasks); }
public async Task <ApiSpecificationSummary> GetSpecificationSummaryById(string specificationId) { Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId)); ApiResponse <ApiSpecificationSummary> specificationSummaryResponse = await _resiliencePolicy.ExecuteAsync(() => _specifications.GetSpecificationSummaryById(specificationId)); return(specificationSummaryResponse?.Content); }
private async Task <IEnumerable <IndexError> > IndexDatasetDefinition(DatasetDefinition definition, PoliciesApiModels.FundingStream fundingStream) { // Calculate hash for model to see if there are changes string modelJson = JsonConvert.SerializeObject(definition); string hashCode = ""; using (SHA256 sha256Generator = SHA256Managed.Create()) { byte[] modelBytes = UTF8Encoding.UTF8.GetBytes(modelJson); foreach (byte hashByte in sha256Generator.ComputeHash(modelBytes)) { hashCode += string.Format("{0:X2}", hashByte); } } DatasetDefinitionIndex datasetDefinitionIndex = new DatasetDefinitionIndex() { Id = definition.Id, Name = definition.Name, Description = definition.Description, LastUpdatedDate = DateTimeOffset.Now, ProviderIdentifier = definition.TableDefinitions.FirstOrDefault()?.FieldDefinitions?.Where(f => f.IdentifierFieldType.HasValue)?.Select(f => Enum.GetName(typeof(IdentifierFieldType), f.IdentifierFieldType.Value)).FirstOrDefault(), ModelHash = hashCode, FundingStreamId = fundingStream.Id, FundingStreamName = fundingStream.Name }; if (string.IsNullOrWhiteSpace(datasetDefinitionIndex.ProviderIdentifier)) { datasetDefinitionIndex.ProviderIdentifier = "None"; } bool updateIndex = true; // Only update index if metadata or model has changed, this is to preserve the LastUpdateDate DatasetDefinitionIndex existingIndex = await _datasetDefinitionSearchRepositoryPolicy.ExecuteAsync(() => _datasetDefinitionSearchRepository.SearchById(definition.Id)); if (existingIndex != null) { if (existingIndex.ModelHash == hashCode && existingIndex.Description == definition.Description && existingIndex.Name == definition.Name && existingIndex.ProviderIdentifier == datasetDefinitionIndex.ProviderIdentifier) { updateIndex = false; } } if (updateIndex) { return(await _datasetDefinitionSearchRepositoryPolicy.ExecuteAsync(() => _datasetDefinitionSearchRepository.Index(new DatasetDefinitionIndex[] { datasetDefinitionIndex }))); } return(Enumerable.Empty <IndexError>()); }
public async Task <IActionResult> GetFundingStreams() { ApiResponse <IEnumerable <PolicyApiClientModel.FundingStream> > fundingStreamsApiResponse = await _policiesApiClientPolicy.ExecuteAsync(() => _policiesApiClient.GetFundingStreams()); IEnumerable <Models.FundingStream> fundingStreams = _mapper.Map <IEnumerable <Models.FundingStream> >(fundingStreamsApiResponse.Content); return(new OkObjectResult(fundingStreams)); }
public Task RegisterBusHandlerAsync(IRequestMatcher pattern, Func <Message, CancellationToken, Task> handler, RouteExecution execution = RouteExecution.Asynchronous, RouteMode mode = RouteMode.Observe, CancellationToken cancellationToken = default) { return(_policy.ExecuteAsync((_, registerPolicyToken) => _registration.RegisterBusHandlerAsync( pattern, (message, handlerToken) => _policy.ExecuteAsync((__, handlerPolicyToken) => handler(message, handlerPolicyToken), new Context(), handlerToken), execution, mode, registerPolicyToken ), new Context(), cancellationToken)); }
private async Task CacheJob(Job job) { // don't cache the job if it's been superseded or there is no associated specification if (job.CompletionStatus == CompletionStatus.Superseded || string.IsNullOrWhiteSpace(job.SpecificationId)) { return; } string cacheKey = $"{CacheKeys.LatestJobs}{job.SpecificationId}:{job.JobDefinitionId}"; await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync(cacheKey, job)); }
private async Task <TEntity> QuerySingle <TEntity>(string sql, CommandType commandType, object parameters) { using IDbConnection connection = NewOpenConnection(); return(await _queryAsyncPolicy.ExecuteAsync(() => connection.QuerySingleOrDefaultAsync <TEntity>(sql, parameters ?? new { }, commandType: commandType))); }
private static async Task DownloadItem(KeyValuePair<string, string> item, Uri uri) { try { Logger.Log($"Starting download for {uri}").FireAndForget(); bool isSmall = false; using (var client = new HttpClient()) { HttpRequestMessage m = new HttpRequestMessage(HttpMethod.Head, uri); HttpResponseMessage resp = await client.SendAsync(m); if (resp.Content.Headers.ContentLength < 100) { isSmall = true; } if (resp.Content.Headers.ContentType != null && (resp.Content.Headers.ContentType.ToString() == "text/plain" || resp.Content.Headers.ContentType.ToString() == "text/json")) { isSmall = true; } m.Dispose(); } Directory.CreateDirectory(Path.GetDirectoryName(NormalizePath(item.Key))); try { if (isSmall) { DownloadService ds = new DownloadService(downloadOptSmall); await RetryPolicy.ExecuteAsync(async () => { await ds.DownloadFileAsync(uri.ToString(), item.Key); }); } else { await RetryPolicy.ExecuteAsync(async () => { await new DownloadService(downloadOpt).DownloadFileAsync(uri.ToString(), item.Key); }); } } catch (System.IO.InvalidDataException) { await RetryPolicy.ExecuteAsync(async () => { await DownloadManually(uri, item.Key); }); } Logger.Log($"Download for {uri} finished.").FireAndForget(); } catch (System.Net.WebException e) { if (e.Message == "The remote server returned an error: (404) Not Found.") { Logger.Log("Could not download " + uri + ": 404 not found.", LogType.ERROR).FireAndForget(); } else { throw; } } }
private async Task <(Stream assembly, string etag)> GetAssemblyFromBlobStorage(string specificationId) { ICloudBlob assemblyBlob = await GetAssemblyCloudBlob(specificationId); if (assemblyBlob == null) { return(null, null); } Stream assembly = await _blobResilience.ExecuteAsync(() => _blobs.DownloadToStreamAsync(assemblyBlob)); return(assembly, assemblyBlob.Properties.ETag); }
/// <summary> /// Processes download queue by downloading each /// file to download success location. /// </summary> /// <param name="cancellationToken">Cancellation token</param> /// <returns> /// Task object for continuation /// </returns> /// <exception cref="System.Exception"></exception> private async Task ProcessDownloadQueue(CancellationToken cancellationToken) { var fileCount = 0; while (DownloadQueue.TryDequeue(out DataMessage dataMessage)) { var response = await _httpClientHelper.GetRequestAsync(new UriBuilder(dataMessage.DownloadLocation).Uri, true, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); if (!response.IsSuccessStatusCode) { throw new JobExecutionException(string.Format(Resources.Job_0_Download_failure_1, _context.JobDetail.Key, response.StatusCode)); } using (var downloadedStream = await response.Content.ReadAsStreamAsync()) { if (fileCount > 0 && _settings.DelayBetweenFiles > 0) //Only delay after first file and never after last. { System.Threading.Thread.Sleep(TimeSpan.FromSeconds(_settings.DelayBetweenFiles)); } fileCount++; cancellationToken.ThrowIfCancellationRequested(); //Downloaded file has no file name. We need to create it. //It will be timestamp followed by number in this download batch. var fileName = $"{DateTime.Now:yyyy-MM-dd_HH-mm-ss-ffff}-{fileCount:D6}"; fileName = Path.ChangeExtension(fileName, "zip"); dataMessage.FullPath = Path.Combine(_settings.DownloadSuccessDir, fileName); dataMessage.Name = fileName; dataMessage.MessageStatus = MessageStatus.Succeeded; await _retryPolicyForAsyncIo.ExecuteAsync(ct => FileOperationsHelper.CreateAsync(downloadedStream, dataMessage.FullPath, ct), cancellationToken); cancellationToken.ThrowIfCancellationRequested(); } if (_settings.LogVerbose || Log.IsDebugEnabled) { Log.DebugFormat(CultureInfo.InvariantCulture, string.Format(Resources.Job_0_File_1_was_downloaded, _context.JobDetail.Key, dataMessage.FullPath.Replace(@"{", @"{{").Replace(@"}", @"}}"))); } await AcknowledgeDownload(dataMessage, cancellationToken); if (_settings.UnzipPackage) { await _retryPolicyForAsyncIo.ExecuteAsync(ct => FileOperationsHelper.UnzipPackageAsync(dataMessage.FullPath, _settings.DeletePackage, _settings.AddTimestamp, ct), cancellationToken); cancellationToken.ThrowIfCancellationRequested(); } } }
public async Task <IActionResult> AssignProfilePatternKey( string fundingStreamId, string fundingPeriodId, string providerId, ProfilePatternKey profilePatternKey, Reference author) { Guard.IsNullOrWhiteSpace(fundingStreamId, nameof(fundingStreamId)); Guard.IsNullOrWhiteSpace(fundingPeriodId, nameof(fundingPeriodId)); Guard.IsNullOrWhiteSpace(providerId, nameof(providerId)); Guard.ArgumentNotNull(profilePatternKey, nameof(profilePatternKey)); PublishedProvider publishedProvider = await _publishingResiliencePolicy.ExecuteAsync(async() => await _publishedFundingRepository.GetPublishedProvider(fundingStreamId, fundingPeriodId, providerId)); if (publishedProvider == null) { return(new StatusCodeResult((int)HttpStatusCode.NotFound)); } FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStreamId, fundingPeriodId); if (fundingConfiguration == null || !fundingConfiguration.EnableUserEditableRuleBasedProfiles) { return(new BadRequestObjectResult($"User not allowed to edit rule based profiles for funding stream - '{fundingStreamId}' and funding period - '{fundingPeriodId}'")); } if (MatchingProfilePatternKeyExists(publishedProvider.Current, profilePatternKey)) { return(new StatusCodeResult((int)HttpStatusCode.NotModified)); } PublishedProvider modifiedPublishedProvider = await CreateVersion(publishedProvider, author); if (modifiedPublishedProvider == null) { return(new StatusCodeResult((int)HttpStatusCode.BadRequest)); } PublishedProviderVersion newPublishedProviderVersion = publishedProvider.Current; newPublishedProviderVersion.SetProfilePatternKey(profilePatternKey, author); await ProfileFundingLineValues(newPublishedProviderVersion, profilePatternKey); await _publishedProviderErrorDetection.ProcessPublishedProvider(publishedProvider, _ => _ is FundingLineValueProfileMismatchErrorDetector); await SavePublishedProvider(publishedProvider, newPublishedProviderVersion); return(new StatusCodeResult((int)HttpStatusCode.OK)); }
private async Task RemoveBlob(string name, string container) { try { ICloudBlob blob = await _resilience.ExecuteAsync(() => _blobClient.GetBlobReferenceFromServerAsync(name, container)); await _resilience.ExecuteAsync(() => blob.DeleteIfExistsAsync()); } catch (Exception e) { _logger.Error(e, $"Unable to delete blob {name} from {container}"); } }
public void InterceptSynchronous(IInvocation invocation) { var capture = invocation.CaptureProceedInfo(); invocation.ReturnValue = _asyncPolicy .ExecuteAsync(() => { capture.Invoke(); return(Task.FromResult(invocation.ReturnValue)); }) .ConfigureAwait(false) .GetAwaiter() .GetResult(); }
public async Task <Container> InitializeCollection(CosmosClient client) { Database database = client.GetDatabase(_cosmosDataStoreConfiguration.DatabaseId); Container containerClient = database.GetContainer(_cosmosCollectionConfiguration.CollectionId); _logger.LogInformation("Finding Container: {collectionId}", _cosmosCollectionConfiguration.CollectionId); AsyncPolicy retryPolicy = _retryExceptionPolicyFactory.GetRetryPolicy(); var existingContainer = await retryPolicy.ExecuteAsync(async() => await database.TryGetContainerAsync(_cosmosCollectionConfiguration.CollectionId)); _logger.LogInformation("Creating Cosmos Container if not exits: {collectionId}", _cosmosCollectionConfiguration.CollectionId); ContainerResponse containerResponse = await retryPolicy.ExecuteAsync(async() => await database.CreateContainerIfNotExistsAsync( _cosmosCollectionConfiguration.CollectionId, $"/{KnownDocumentProperties.PartitionKey}", _cosmosCollectionConfiguration.InitialCollectionThroughput)); if (containerResponse.StatusCode == HttpStatusCode.Created || containerResponse.Resource.DefaultTimeToLive != -1) { if (_cosmosCollectionConfiguration.InitialCollectionThroughput.HasValue) { var throughputProperties = ThroughputProperties.CreateManualThroughput(_cosmosCollectionConfiguration.InitialCollectionThroughput.Value); await retryPolicy.ExecuteAsync(async() => await containerClient.ReplaceThroughputAsync(throughputProperties)); } containerResponse.Resource.DefaultTimeToLive = -1; existingContainer = await retryPolicy.ExecuteAsync(async() => await containerClient.ReplaceContainerAsync(containerResponse)); } await retryPolicy.ExecuteAsync(async() => { try { await _clientTestProvider.PerformTest(existingContainer, _cosmosDataStoreConfiguration, _cosmosCollectionConfiguration); } catch (CosmosException e) when(e.StatusCode == HttpStatusCode.TooManyRequests) { // This is the very first interaction with the collection, and we might get this exception // when it calls GetCachedContainerPropertiesAsync, which does not use our request handler. throw new RequestRateExceededException(e.RetryAfter); } }); await _upgradeManager.SetupContainerAsync(containerClient); return(existingContainer); }
private async Task <IEnumerable <FundingConfiguration> > GetFundingConfigurationsByFundingStreamId(string fundingStreamId) { ApiResponse <IEnumerable <FundingConfiguration> > fundingConfigResponse = await _policiesApiClientPolicy.ExecuteAsync(() => _policiesApiClient.GetFundingConfigurationsByFundingStreamId(fundingStreamId)); if (!fundingConfigResponse.StatusCode.IsSuccess()) { string errorMessage = $"Unable to retrieve funding configs for funding stream with ID: {fundingStreamId}"; _logger.Error(errorMessage); return(Enumerable.Empty <FundingConfiguration>()); } return(fundingConfigResponse.Content); }
/// <inheritdoc /> public async Task AddOrReplaceFileMetaToTransactionAsync( FileMeta fileMeta, string transactionId, string fileId, CancellationToken cancellationToken = default) { await retryPolicy .ExecuteAsync( ct => client.AddOrReplaceFileMetaToTransactionAsync( fileMeta, transactionId, fileId, ct), cancellationToken); }