private async Task <bool> HasInvalidStatusCode( LastChangedRecord record, RedisStore redisStore, int responseStatusCode, string requestUrl, HttpResponseMessage response) { if (_validStatusCodes.Contains(responseStatusCode)) { return(false); } _logger.LogWarning( "Backend call to {RequestUrl} ({AcceptType}) returned statuscode {StatusCode} which was invalid.", requestUrl, record.AcceptType, response.StatusCode); record.ErrorCount++; record.LastError = DateTimeOffset.UtcNow; record.LastErrorMessage = $"Backend call to {requestUrl} ({record.AcceptType}) returned statuscode {response.StatusCode} which was invalid."; if (record.ErrorCount >= _maxErrorCount) { _logger.LogInformation( "{CacheKey} reached {MaxErrorCount} errors, purging from cache.", record.CacheKey.ToLowerInvariant(), record.ErrorCount); await redisStore.DeleteKeyAsync(record.CacheKey.ToLowerInvariant()); } return(true); }
public async Task ThenAllUnpopulatedRecordsAreReturned() { var allDbRecords = Context.LastChangedList.ToList(); Assert.Equal(Records.Count, allDbRecords.Count); _populatedRecord = allDbRecords.FirstOrDefault(r => r.Position == r.LastPopulatedPosition); Assert.NotNull(_populatedRecord); var unpopulatedRecords = await _sut.GetUnpopulatedRecordsAsync(1000, DateTimeOffset.UtcNow, CancellationToken.None); Assert.NotEmpty(unpopulatedRecords); Assert.Equal(2, unpopulatedRecords.Count); Assert.True(unpopulatedRecords.TrueForAll(r => r.Position > r.LastPopulatedPosition)); Assert.DoesNotContain(unpopulatedRecords, r => r.Id == _populatedRecord.Id); }
private async Task UpdateRecordInRedisAsync(LastChangedRecord record, RedisStore redisStore, CancellationToken cancellationToken) { var requestUrl = GetBaseUri(record.Uri) + record.Uri; using var response = await _httpClient.GetAsync(requestUrl, record.AcceptType ?? "application/json", cancellationToken); var responseStatusCode = (int)response.StatusCode; if (await HasInvalidStatusCode(record, redisStore, responseStatusCode, requestUrl, response)) { return; } if (await EligibleForDeletion(record, redisStore, responseStatusCode, requestUrl, response)) { return; } var responseContent = await response.Content.ReadAsStringAsync(cancellationToken); var responseHeaders = new Dictionary <string, string[]>(StringComparer.OrdinalIgnoreCase); foreach (var headerToStore in _headersToStore.Concat(new[] { HeaderNames.ETag })) { var headerName = headerToStore.ToLowerInvariant(); if (response.Headers.TryGetValues(headerName, out var headerValues)) { responseHeaders.Add(headerName, headerValues.ToArray()); } } await redisStore.SetAsync( record.CacheKey?.ToLowerInvariant(), responseContent, responseStatusCode, responseHeaders, record.Position); record.LastPopulatedPosition = record.Position; }
private async Task UpdateRecordInRedisAsync(LastChangedRecord record, RedisStore redisStore, CancellationToken cancellationToken) { var requestUrl = _apiBaseAddress + record.Uri; using (var response = await _httpClient.GetAsync(requestUrl, record.AcceptType, cancellationToken)) { var responseStatusCode = (int)response.StatusCode; if (await HasInvalidStatusCode(record, redisStore, responseStatusCode, requestUrl, response)) { return; } if (await EligibleForDeletion(record, redisStore, responseStatusCode, requestUrl, response)) { return; } var responseContent = await response.Content.ReadAsStringAsync(); var responseHeaders = new Dictionary <string, string[]>(); foreach (var headerToStore in _headersToStore) { var headerName = headerToStore.ToLowerInvariant(); if (response.Headers.TryGetValues(headerName, out var headerValues)) { responseHeaders.Add(headerName, headerValues.ToArray()); } } await redisStore.SetAsync( record.CacheKey.ToLowerInvariant(), responseContent, responseStatusCode, responseHeaders); record.LastPopulatedPosition = record.Position; } }
protected async Task <IEnumerable <LastChangedRecord> > GetLastChangedRecordsAndUpdatePosition( string identifier, long position, LastChangedListContext context, CancellationToken cancellationToken) { context.Database.SetCommandTimeout(_commandTimeoutInSeconds); var attachedRecords = new List <LastChangedRecord>(); // Create a record for every type that our API accepts. foreach (var acceptType in _supportedAcceptTypes) { var shortenedApplicationType = acceptType.ToString().ToLowerInvariant(); var id = $"{identifier}.{shortenedApplicationType}"; var record = await context .LastChangedList .FindAsync(id, cancellationToken : cancellationToken); if (record == null) { record = new LastChangedRecord { Id = id, CacheKey = string.Format(CacheKeyFormat, identifier, shortenedApplicationType), Uri = string.Format(UriFormat, identifier), AcceptType = GetApplicationType(acceptType) }; await context.LastChangedList.AddAsync(record, cancellationToken); } record.Position = position; attachedRecords.Add(record); } return(attachedRecords); }
private async Task <bool> EligibleForDeletion( LastChangedRecord record, RedisStore redisStore, int responseStatusCode, string requestUrl, HttpResponseMessage response) { if (!_validStatusCodesToDelete.Contains(responseStatusCode)) { return(false); } _logger.LogInformation( "Backend call to {RequestUrl} ({AcceptType}) returned statuscode {StatusCode} which is eligible for deletion. ({CacheKey})", requestUrl, record.AcceptType, response.StatusCode, record.CacheKey.ToLowerInvariant()); await redisStore.DeleteKeyAsync(record.CacheKey.ToLowerInvariant()); record.LastPopulatedPosition = record.Position; return(true); }