private TemporaryExposureKeyGatewayBatchProtoDto MapBatchDtoToProtoAndSortForSigning(TemporaryExposureKeyGatewayBatchDto dto)
{
var protoKeys = dto.Keys.Select(gatewayDto => _mapper.Map <TemporaryExposureKeyGatewayDtoProto>(gatewayDto)).ToList();
var protoBatch = new TemporaryExposureKeyGatewayBatchProtoDto();
protoBatch.Keys.AddRange(protoKeys);
return(protoBatch);
}
private HttpResponseMessage CreateGatewayResponseFromBatch(IEnumerable <TemporaryExposureKey> keyPackage)
{
var keys = keyPackage.Select(entityKey => _autoMapper.Map <TemporaryExposureKeyGatewayDto>(entityKey)).ToList();
var gatewayProtoKeys = keys.Select(key =>
_autoMapper.Map <FederationGatewayApi.Models.Proto.TemporaryExposureKeyGatewayDto>(key));
var gatewayBatchProto = new FederationGatewayApi.Models.Proto.TemporaryExposureKeyGatewayBatchDto()
{
Keys = { gatewayProtoKeys }
};
return(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(gatewayBatchProto.ToString())
});
}
private bool TrySendKeyBatchToTheGateway(TemporaryExposureKeyGatewayBatchProtoDto protoBatch, SortOrder keySortOrderForSignature)
{
var upoadKeysEndpointUrl = _euGatewayConfig.UrlNormalized + EuGatewayContract.Endpoint.KeysUploadEndpoint;
var batchBytes = protoBatch.ToByteArray();
//sign
var signing = _signatureService.Sign(protoBatch, keySortOrderForSignature);
var signingBase64 = _encodingService.EncodeToBase64(signing);
var body = new ByteArrayContent(batchBytes);
var uniqueTag = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString();
body.Headers.Add("batchTag", uniqueTag);
body.Headers.Add("Content-Type", "application/protobuf;version=1.0");
body.Headers.Add("batchSignature", signingBase64);
var response = _gatewayHttpClient.PostAsync(upoadKeysEndpointUrl, body).Result;
var message = response.Content.ReadAsStringAsync().Result;
var code = response.StatusCode;
switch (code)
{
case System.Net.HttpStatusCode.OK:
_logger.LogInformation($"Response - OK. Message: {message}");
var containsHtml = message.Contains("<body>");
_logger.LogError($"UeGateway response with code 200 with HTML in the response: {containsHtml}. UeGateway Server is down!");
// Server is down - https://github.com/eu-federation-gateway-service/efgs-federation-gateway/issues/151
break;
case System.Net.HttpStatusCode.Created:
_logger.LogInformation("Keys successfully uploaded.");
return(true);
case System.Net.HttpStatusCode.MultiStatus:
_logger.LogWarning($"Data partially added with warnings: {message}.");
return(true);
case System.Net.HttpStatusCode.BadRequest:
_logger.LogError($"Bad request: {message}");
break;
case System.Net.HttpStatusCode.Forbidden:
_logger.LogError($"Forbidden call in cause of missing or invalid client certificate. Message: {message}");
break;
case System.Net.HttpStatusCode.NotAcceptable:
_logger.LogError($"Data format or content is not valid. Massage:{message}");
break;
case System.Net.HttpStatusCode.Conflict:
_logger.LogError($"Data already exist. Message: {message}");
break;
case System.Net.HttpStatusCode.RequestEntityTooLarge:
_logger.LogError($"Payload to large. Message: {message}");
break;
case System.Net.HttpStatusCode.InternalServerError:
_logger.LogError($"Are not able to write data. Retry please. Message: {message}");
break;
default:
_logger.LogError($"Response code was not recognized. Status code: {code}, message: {message}");
break;
}
return(false);
}
private BatchStatus UploadNextBatch(int batchSize, int keyAgeLimitInDays)
{
var lastSyncState = _settingsService.GetGatewayUploadState();
// Select only keys from last N days (by date of record creation)
var uploadedOnAndAfterTicks = lastSyncState.CreationDateOfLastUploadedKey;
var uploadedOnAndAfter = uploadedOnAndAfterTicks.HasValue ? new DateTime(uploadedOnAndAfterTicks.Value, DateTimeKind.Utc) : DateTime.UnixEpoch;
int batchSizePlusOne = batchSize + 1; // if it will return n + 1 then there is at last one more records to send
// Get key package - collection of the records created (uploaded by mobile app) in the db after {uploadedOn}
IList <TemporaryExposureKey> keyPackage = _tempKeyRepository.GetKeysOnlyFromApiOriginCountryUploadedAfterTheDateForGatewayUpload(
uploadedOnAndLater: uploadedOnAndAfter,
numberOfRecordToSkip: lastSyncState.NumberOfKeysProcessedFromTheLastCreationDate,
maxCount: batchSizePlusOne,
new KeySource[] { KeySource.SmitteStopApiVersion2 });
// Take all record uploaded after the date.
var currBatchStatus = new BatchStatus()
{
NextBatchExists = keyPackage.Count == batchSizePlusOne
};
keyPackage = keyPackage.Take(batchSize).ToList();
currBatchStatus.KeysProcessed = keyPackage.Count;
currBatchStatus.ProcessedSuccessfully = true;
if (!keyPackage.Any())
{
_logger.LogInformation("KeyPackage is empty. Stopping the upload process.");
return(currBatchStatus);
}
_logger.LogInformation($"{keyPackage.Count} records picked.");
var oldestKeyFromPackage = keyPackage.Last(); // be aware that it could not be present in the batch. This is the last record (have oldest CreationOn) that has been processed.
// Create Batch based on package but filter in on RollingStartNumber. Batch.Size <= Package.Size
// We can send data not older than N days ago (age of the key save in key.RollingStartNumber)
DateTime createdAfter = DateTime.UtcNow.AddDays(-keyAgeLimitInDays);
var createdAfterTimestamp = _epochConverter.ConvertToEpoch(createdAfter);
var filteredKeysForBatch =
keyPackage
.Where(k => k.RollingStartNumber > createdAfterTimestamp)
.Where(k => k.DaysSinceOnsetOfSymptoms >= KeyValidator.DaysSinceOnsetOfSymptomsValidRangeMin)
.Where(k => k.DaysSinceOnsetOfSymptoms <= KeyValidator.DaysSinceOnsetOfSymptomsValidRangeMax)
.ToList();
currBatchStatus.KeysToSend = filteredKeysForBatch.Count;
var batch = CreateGatewayBatchFromKeys(filteredKeysForBatch);
TemporaryExposureKeyGatewayBatchProtoDto protoBatch = MapBatchDtoToProtoAndSortForSigning(batch);
_logger.LogInformation($"{currBatchStatus.KeysToSend} records to send after filtering by age.");
if (protoBatch.Keys.Any())
{
_logger.LogInformation($"Sending batch...");
currBatchStatus.ProcessedSuccessfully = TrySendKeyBatchToTheGateway(protoBatch, KeysSortOrderUsedToCreateAndVerifyUploadRequestSignature);
}
else
{
_logger.LogInformation($"Nothing to sent for this package. All picked keys are older then {keyAgeLimitInDays} days or have incorrect value of DaysSinceOnsetOfSymptoms.");
}
if (currBatchStatus.ProcessedSuccessfully)
{
currBatchStatus.KeysSent = protoBatch.Keys.Count;
var currSyncState = new GatewayUploadState();
var lastSentKeyTicks = oldestKeyFromPackage.CreatedOn.Ticks;
// If the date of the last sent element (recently) has changed in compare the date from the previous batch?
// NO - Update the state: Do not change the date, only offset (+=)
// YES - Update the state: Change date to the date from last sent element. Update the index to the offset from the first appearance of that date.)
if (lastSyncState.CreationDateOfLastUploadedKey == lastSentKeyTicks)
{
currSyncState.CreationDateOfLastUploadedKey = lastSyncState.CreationDateOfLastUploadedKey;
currSyncState.NumberOfKeysProcessedFromTheLastCreationDate = lastSyncState.NumberOfKeysProcessedFromTheLastCreationDate + currBatchStatus.KeysProcessed;
}
else
{
currSyncState.CreationDateOfLastUploadedKey = oldestKeyFromPackage.CreatedOn.Ticks;
// find offset form fist element with CreationDateOfLastUploadedKey to last sent element
var indexOfTheFirstKeyWithTheDate = keyPackage.Select((k, i) => new { Key = k, Index = i })
.First(o => o.Key.CreatedOn.Ticks == lastSentKeyTicks)
.Index;
currSyncState.NumberOfKeysProcessedFromTheLastCreationDate = currBatchStatus.KeysProcessed - indexOfTheFirstKeyWithTheDate;
}
// save new sync status
_settingsService.SaveGatewaySyncState(currSyncState);
}
else
{
_logger.LogError($"Error sending the batch! Stopping upload process.");
}
return(currBatchStatus);
}
