private async Task <bool> HasInvalidStatusCode(
LastChangedRecord record,
RedisStore redisStore,
int responseStatusCode,
string requestUrl,
HttpResponseMessage response)
{
if (_validStatusCodes.Contains(responseStatusCode))
{
return(false);
}
_logger.LogWarning(
"Backend call to {RequestUrl} ({AcceptType}) returned statuscode {StatusCode} which was invalid.",
requestUrl,
record.AcceptType,
response.StatusCode);
record.ErrorCount++;
record.LastError = DateTimeOffset.UtcNow;
record.LastErrorMessage = $"Backend call to {requestUrl} ({record.AcceptType}) returned statuscode {response.StatusCode} which was invalid.";
if (record.ErrorCount >= _maxErrorCount)
{
_logger.LogInformation(
"{CacheKey} reached {MaxErrorCount} errors, purging from cache.",
record.CacheKey.ToLowerInvariant(),
record.ErrorCount);
await redisStore.DeleteKeyAsync(record.CacheKey.ToLowerInvariant());
}
return(true);
}
public async Task ThenAllUnpopulatedRecordsAreReturned()
{
var allDbRecords = Context.LastChangedList.ToList();
Assert.Equal(Records.Count, allDbRecords.Count);
_populatedRecord = allDbRecords.FirstOrDefault(r => r.Position == r.LastPopulatedPosition);
Assert.NotNull(_populatedRecord);
var unpopulatedRecords = await _sut.GetUnpopulatedRecordsAsync(1000, DateTimeOffset.UtcNow, CancellationToken.None);
Assert.NotEmpty(unpopulatedRecords);
Assert.Equal(2, unpopulatedRecords.Count);
Assert.True(unpopulatedRecords.TrueForAll(r => r.Position > r.LastPopulatedPosition));
Assert.DoesNotContain(unpopulatedRecords, r => r.Id == _populatedRecord.Id);
}
private async Task UpdateRecordInRedisAsync(LastChangedRecord record, RedisStore redisStore, CancellationToken cancellationToken)
{
var requestUrl = GetBaseUri(record.Uri) + record.Uri;
using var response = await _httpClient.GetAsync(requestUrl, record.AcceptType ?? "application/json", cancellationToken);
var responseStatusCode = (int)response.StatusCode;
if (await HasInvalidStatusCode(record, redisStore, responseStatusCode, requestUrl, response))
{
return;
}
if (await EligibleForDeletion(record, redisStore, responseStatusCode, requestUrl, response))
{
return;
}
var responseContent = await response.Content.ReadAsStringAsync(cancellationToken);
var responseHeaders = new Dictionary <string, string[]>(StringComparer.OrdinalIgnoreCase);
foreach (var headerToStore in _headersToStore.Concat(new[] { HeaderNames.ETag }))
{
var headerName = headerToStore.ToLowerInvariant();
if (response.Headers.TryGetValues(headerName, out var headerValues))
{
responseHeaders.Add(headerName, headerValues.ToArray());
}
}
await redisStore.SetAsync(
record.CacheKey?.ToLowerInvariant(),
responseContent,
responseStatusCode,
responseHeaders,
record.Position);
record.LastPopulatedPosition = record.Position;
}
private async Task UpdateRecordInRedisAsync(LastChangedRecord record, RedisStore redisStore, CancellationToken cancellationToken)
{
var requestUrl = _apiBaseAddress + record.Uri;
using (var response = await _httpClient.GetAsync(requestUrl, record.AcceptType, cancellationToken))
{
var responseStatusCode = (int)response.StatusCode;
if (await HasInvalidStatusCode(record, redisStore, responseStatusCode, requestUrl, response))
{
return;
}
if (await EligibleForDeletion(record, redisStore, responseStatusCode, requestUrl, response))
{
return;
}
var responseContent = await response.Content.ReadAsStringAsync();
var responseHeaders = new Dictionary <string, string[]>();
foreach (var headerToStore in _headersToStore)
{
var headerName = headerToStore.ToLowerInvariant();
if (response.Headers.TryGetValues(headerName, out var headerValues))
{
responseHeaders.Add(headerName, headerValues.ToArray());
}
}
await redisStore.SetAsync(
record.CacheKey.ToLowerInvariant(),
responseContent,
responseStatusCode,
responseHeaders);
record.LastPopulatedPosition = record.Position;
}
}
protected async Task <IEnumerable <LastChangedRecord> > GetLastChangedRecordsAndUpdatePosition(
string identifier,
long position,
LastChangedListContext context,
CancellationToken cancellationToken)
{
context.Database.SetCommandTimeout(_commandTimeoutInSeconds);
var attachedRecords = new List <LastChangedRecord>();
// Create a record for every type that our API accepts.
foreach (var acceptType in _supportedAcceptTypes)
{
var shortenedApplicationType = acceptType.ToString().ToLowerInvariant();
var id = $"{identifier}.{shortenedApplicationType}";
var record = await context
.LastChangedList
.FindAsync(id, cancellationToken : cancellationToken);
if (record == null)
{
record = new LastChangedRecord
{
Id = id,
CacheKey = string.Format(CacheKeyFormat, identifier, shortenedApplicationType),
Uri = string.Format(UriFormat, identifier),
AcceptType = GetApplicationType(acceptType)
};
await context.LastChangedList.AddAsync(record, cancellationToken);
}
record.Position = position;
attachedRecords.Add(record);
}
return(attachedRecords);
}
private async Task <bool> EligibleForDeletion(
LastChangedRecord record,
RedisStore redisStore,
int responseStatusCode,
string requestUrl,
HttpResponseMessage response)
{
if (!_validStatusCodesToDelete.Contains(responseStatusCode))
{
return(false);
}
_logger.LogInformation(
"Backend call to {RequestUrl} ({AcceptType}) returned statuscode {StatusCode} which is eligible for deletion. ({CacheKey})",
requestUrl,
record.AcceptType,
response.StatusCode,
record.CacheKey.ToLowerInvariant());
await redisStore.DeleteKeyAsync(record.CacheKey.ToLowerInvariant());
record.LastPopulatedPosition = record.Position;
return(true);
}
