public void PullKeysParams_PullFirstTime_ConsentGiven()
{
//Given the app never pulled before, but consent was given to pull EU keys.
_preferences.Clear();
OnboardingStatusHelper.Status = OnboardingStatus.CountriesOnboardingCompleted;
//When the app pulls for the first time
PullKeysParams pullParams = PullKeysParams.GenerateParams();
//We will request all keys for today but only NO keys (consent was not given yet)
Assert.Equal(SystemTime.Now().Date, pullParams.Date);
Assert.Equal(1, pullParams.BatchNumber);
Assert.Equal(BatchType.ALL, pullParams.BatchType);
//Reset
SystemTime.ResetDateTime();
}
public void PullKeysParams_PullFirstTime()
{
//Given the app never pulled before (preferences will be empty)
_preferences.Clear();
DateTime today = new DateTime(2020, 4, 4, 8, 8, 8);
SystemTime.SetDateTime(today);
//When the app pulls for the first time
PullKeysParams pullParams = PullKeysParams.GenerateParams();
//We will request all keys for today
Assert.Equal(today.Date, pullParams.Date);
Assert.Equal(1, pullParams.BatchNumber);
Assert.Equal(BatchType.ALL, pullParams.BatchType);
//Reset
SystemTime.ResetDateTime();
}
public void PullKeysParams_LastPullWas15DaysAgo()
{
//Given that last time we pulled keys was 20 days ago and the last batch was number 2 for that day
DateTime lastPullDate = SystemTime.Now().AddDays(-20).Date;
_preferences.Clear();
_preferences.Set(PULL_DATE_KEY, lastPullDate.ToUniversalTime());
_preferences.Set(PULL_BATCH_KEY, 2);
//When the app pulls
PullKeysParams pullParams = PullKeysParams.GenerateParams();
//Then it pulls only for the last 14 days, incl. today.
Assert.Equal(SystemTime.Now().AddDays(-13).Date, pullParams.Date);
Assert.Equal(1, pullParams.BatchNumber);
Assert.Equal(BatchType.ALL, pullParams.BatchType);
//Reset
SystemTime.ResetDateTime();
}
public void PullKeysParams_NotFirstPullOfDay()
{
//Given that we previously successfully pulled keys today and the last batch was number 3.
DateTime lastPullDate = SystemTime.Now().Date;
_preferences.Clear();
_preferences.Set(PULL_DATE_KEY, lastPullDate.ToUniversalTime());
_preferences.Set(PULL_BATCH_KEY, 3);
//When the app pulls
PullKeysParams pullParams = PullKeysParams.GenerateParams();
//Then the next batch number for today will be requested
Assert.Equal(lastPullDate, pullParams.Date);
Assert.Equal(4, pullParams.BatchNumber);
Assert.Equal(BatchType.NO, pullParams.BatchType);
//Reset
SystemTime.ResetDateTime();
}
public void PullKeysParams_LastPullWasThreeDaysAgo()
{
//Given that last time we pulled keys was three days ago and the last batch was number 2 for that day
DateTime lastPullDate = SystemTime.Now().AddDays(-3).Date;
_preferences.Clear();
_preferences.Set(PULL_DATE_KEY, lastPullDate.ToUniversalTime());
_preferences.Set(PULL_BATCH_KEY, 2);
//When the app pulls
PullKeysParams pullParams = PullKeysParams.GenerateParams();
//Then the next batch number for three days ago will be requested
Assert.Equal(lastPullDate, pullParams.Date);
Assert.Equal(3, pullParams.BatchNumber);
Assert.Equal(BatchType.NO, pullParams.BatchType);
//Reset
SystemTime.ResetDateTime();
}
public void PullKeysParams_NotFirstPull()
{
//App was just updated to pull EU keys, and the last pull was NO keys.
DateTime lastPullDate = SystemTime.Now().Date;
_preferences.Clear();
_preferences.Set(PULL_DATE_KEY, lastPullDate.ToUniversalTime());
_preferences.Set(PULL_BATCH_KEY, 3);
_preferences.Set(PULL_BATCH_TYPE, "no");
//When the app pulls for the first time
PullKeysParams pullParams = PullKeysParams.GenerateParams();
//We will request all keys for today
Assert.Equal(SystemTime.Now().Date, pullParams.Date);
Assert.Equal(1, pullParams.BatchNumber);
Assert.Equal(BatchType.ALL, pullParams.BatchType);
//Reset
SystemTime.ResetDateTime();
}
public void PullKeysParams_NotFirstPull_ConsentGiven()
{
//Given consent was just given to pull EU keys, and the last pull was NO keys.
DateTime lastPullDate = SystemTime.Now().Date;
_preferences.Clear();
OnboardingStatusHelper.Status = OnboardingStatus.CountriesOnboardingCompleted;
_preferences.Set(PULL_DATE_KEY, lastPullDate.ToUniversalTime());
_preferences.Set(PULL_BATCH_KEY, 3);
_preferences.Set(PULL_BATCH_TYPE, "no");
//When the app pulls for the first time
PullKeysParams pullParams = PullKeysParams.GenerateParams();
//We will request all keys for today but only NO keys (consent was not given yet)
Assert.Equal(SystemTime.Now().Date, pullParams.Date);
Assert.Equal(1, pullParams.BatchNumber);
Assert.Equal(BatchType.ALL, pullParams.BatchType);
//Reset
SystemTime.ResetDateTime();
}
public void PullKeysParams_PullAtTimes(string time)
{
//Given that the current time is as follows, and the app never pulled before
DateTime today = SystemTime.Now().Date;
string dateString = today.ToString("yyyy-MM-dd", CultureInfo.GetCultureInfo("nn-NO"));
DateTime newNow = DateTime.ParseExact($"{dateString} {time}", "yyyy-MM-dd HH:mm z", CultureInfo.GetCultureInfo("nn-NO"));
SystemTime.SetDateTime(newNow);
_preferences.Clear();
//When the app pulls
PullKeysParams pullParams = PullKeysParams.GenerateParams();
string pullQueryParams = pullParams.ToBatchFileRequest();
//The date will be parsed right (UTC)
string expectedDateString = SystemTime.Now().ToUniversalTime().Date.ToString("yyyy-MM-dd");
Assert.Equal($"{expectedDateString}_1_all.zip", pullQueryParams);
//Reset
SystemTime.ResetDateTime();
}
public void PullKeysParams_LastPullWasAtTimes(string time, int daysAgo, int expectedDaysAgo)
{
//Given that last time we pulled keys was at given time
DateTime today = SystemTime.Now();
string dateString = today.AddDays(daysAgo).ToString("yyyy-MM-dd", CultureInfo.GetCultureInfo("nn-NO"));
DateTime lastPullDate = DateTime.ParseExact($"{dateString} {time}", "yyyy-MM-dd HH:mm z", CultureInfo.GetCultureInfo("nn-NO"));
_preferences.Clear();
_preferences.Set(PULL_DATE_KEY, lastPullDate.ToUniversalTime());
_preferences.Set(PULL_BATCH_KEY, 2);
//When the app pulls
PullKeysParams pullParams = PullKeysParams.GenerateParams();
string pullQueryParams = pullParams.ToBatchFileRequest();
//The date will be parsed right (UTC time)
string expectedDateString = today.AddDays(expectedDaysAgo).ToString("yyyy-MM-dd");
Assert.Equal($"{expectedDateString}_3_no.zip", pullQueryParams);
//Reset
SystemTime.ResetDateTime();
}
/// <summary>
/// Fetches the new keys if any.
/// </summary>
/// <returns>Paths to the temporary location where the zips are stored</returns>
public async Task <IEnumerable <string> > PullNewKeys(ExposureNotificationWebService service, CancellationToken cancellationToken)
{
PullKeysParams requestParams = PullKeysParams.GenerateParams();
LocalPreferencesHelper.DidFirstFileOfTheDayEndedWith204 = false;
List <string> zipLocations = new List <string>();
bool lastPull = false;
int? lastBatchReceived = null;
int? lastReceivedStatusCodeFromRequest = null;
while (!lastPull)
{
string requestUrl = requestParams.ToBatchFileRequest();
ApiResponse <Stream> response = await service.GetDiagnosisKeys(requestUrl, cancellationToken);
HttpHeaders headers = response.Headers;
lastReceivedStatusCodeFromRequest = response.StatusCode;
bool headersAreValid = true;
if (response == null || (!response.IsSuccessfull))
{
if (response?.StatusCode == 410)
{
NotificationsHelper.CreateNotification(NotificationsEnum.ApiDeprecated, 0);
string warning = "410 Api was deprecated";
_developerTools.AddToPullHistoryRecord(warning, requestUrl);
LogUtils.LogMessage(LogSeverity.WARNING, $"{_logPrefix}.{nameof(DownloadZips)}: {warning}");
}
else
{
//Failed to fetch new keys due to server error. This is already logged in the webservice.
_developerTools.AddToPullHistoryRecord($"{response.StatusCode} Server Error", requestUrl);
}
break; //Abort pulling
}
// If the server says 204: No Content, it means that there were no new keys (I.e. the request batch does not exist)
if (response.StatusCode == 204)
{
if (requestParams.Date.Date < SystemTime.Now().Date)
{
//If there were no new keys for a day which is not today, then move on to fetch keys for the next date.
requestParams.Date = requestParams.Date.AddDays(1);
requestParams.BatchNumber = 1;
lastPull = false;
}
else
{
//There were no new keys to fetch for today
_developerTools.AddToPullHistoryRecord($"204 No Content - No new keys", requestUrl);
string warning = $"API {response.Endpoint} returned 204 No Content - No new keys since last pull";
LogUtils.LogMessage(LogSeverity.WARNING, $"{_logPrefix}.{nameof(DownloadZips)}: {warning}");
lastPull = true;
}
}
else
{
try
{
int lastBatchReceivedValue = int.Parse(headers.GetValues(LastBatchReturnedHeader).First());
bool moreBatchesExist = bool.Parse(headers.GetValues(MoreBatchesExistHeader).First());
//If both headers parse (no exceptions), then save lastBatchValue to be persisted
lastBatchReceived = lastBatchReceivedValue;
if (moreBatchesExist)
{
//There are still more batches to fetch for the given date
requestParams.BatchNumber = (int)lastBatchReceived + 1;
lastPull = false;
}
else if (requestParams.Date.Date < SystemTime.Now().Date)
{
//If there were no new keys for a day which is not today, then move on to fetch keys for the next date.
requestParams.Date = requestParams.Date.AddDays(1);
requestParams.BatchNumber = 1;
lastPull = false;
}
else
{
//There are no more batches to fetch for today. Try again in some hours.
lastPull = true;
}
}
catch (Exception e)
{
headersAreValid = false;
HandleErrorWhenPulling(e, $"Failed to parse {MoreBatchesExistHeader} or {LastBatchReturnedHeader} header.", requestUrl);
break; //Abort pulling
}
}
// Copy the zip stream in the response into a temp file
if (response.StatusCode == 200 && headersAreValid)
{
try
{
_developerTools.AddToPullHistoryRecord("200 OK", requestUrl);
string tmpFile = Path.Combine(ServiceLocator.Current.GetInstance <IFileSystem>().CacheDirectory, Guid.NewGuid() + ".zip");
FileStream tmpFileStream = File.Create(tmpFile);
await response.Data.CopyToAsync(tmpFileStream);
tmpFileStream.Close();
zipLocations.Add(tmpFile);
}
catch (Exception e)
{
HandleErrorWhenPulling(e, "Failed to save zip locally", requestUrl);
break; //Abort pulling
}
}
}
if (zipLocations.Any() && lastBatchReceived != null)
{
//Persist the last batch that was fetched, to know which one to fetch next time the background task runs.
LocalPreferencesHelper.LastPullKeysBatchNumberNotSubmitted = (int)lastBatchReceived;
//Also save the last batchtype fetched
LocalPreferencesHelper.LastPulledBatchType = requestParams.BatchType;
}
// Edge case for when pulling across multiple days ends up in 204 for the first file
if (requestParams.Date.Date == SystemTime.Now().Date &&
requestParams.BatchNumber == 1 &&
lastReceivedStatusCodeFromRequest == 204)
{
LocalPreferencesHelper.DidFirstFileOfTheDayEndedWith204 = true;
}
return(zipLocations);
}
public async void PullKeys_NewTermsWereApproved()
{
int lastBatchNumNO = 4;
int lastBatchNumALL = 5;
//Both NO and EU keys exist for the last 3 days
ExposureNotificationWebService mockedService = _helper.MockedService(new List <PullKeysMockData>
{
new PullKeysMockData(day1, 1, BatchType.NO).HttpStatusCode(200).WithLastBatchHeader(1).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day1, 2, BatchType.NO).HttpStatusCode(200).WithLastBatchHeader(2).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day1, 3, BatchType.NO).HttpStatusCode(200).WithLastBatchHeader(3).WithMoreBatchesExistHeader(false),
new PullKeysMockData(day1, 4, BatchType.NO).HttpStatusCode(204),
new PullKeysMockData(day1, 1, BatchType.ALL).HttpStatusCode(200).WithLastBatchHeader(1).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day1, 2, BatchType.ALL).HttpStatusCode(200).WithLastBatchHeader(2).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day1, 3, BatchType.ALL).HttpStatusCode(200).WithLastBatchHeader(3).WithMoreBatchesExistHeader(false),
new PullKeysMockData(day1, 4, BatchType.ALL).HttpStatusCode(204),
new PullKeysMockData(day2, 1, BatchType.NO).HttpStatusCode(200).WithLastBatchHeader(1).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day2, 2, BatchType.NO).HttpStatusCode(200).WithLastBatchHeader(2).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day2, 3, BatchType.NO).HttpStatusCode(200).WithLastBatchHeader(3).WithMoreBatchesExistHeader(false),
new PullKeysMockData(day2, 4, BatchType.NO).HttpStatusCode(204),
new PullKeysMockData(day2, 1, BatchType.ALL).HttpStatusCode(200).WithLastBatchHeader(1).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day2, 2, BatchType.ALL).HttpStatusCode(200).WithLastBatchHeader(2).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day2, 3, BatchType.ALL).HttpStatusCode(200).WithLastBatchHeader(3).WithMoreBatchesExistHeader(false),
new PullKeysMockData(day2, 4, BatchType.ALL).HttpStatusCode(204),
new PullKeysMockData(day3, 1, BatchType.NO).HttpStatusCode(200).WithLastBatchHeader(1).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day3, 2, BatchType.NO).HttpStatusCode(200).WithLastBatchHeader(2).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day3, 3, BatchType.NO).HttpStatusCode(200).WithLastBatchHeader(lastBatchNumNO).WithMoreBatchesExistHeader(false),
new PullKeysMockData(day3, 4, BatchType.NO).HttpStatusCode(204),
new PullKeysMockData(day3, 5, BatchType.NO).HttpStatusCode(204),
new PullKeysMockData(day3, 1, BatchType.ALL).HttpStatusCode(200).WithLastBatchHeader(1).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day3, 2, BatchType.ALL).HttpStatusCode(200).WithLastBatchHeader(2).WithMoreBatchesExistHeader(true),
new PullKeysMockData(day3, 3, BatchType.ALL).HttpStatusCode(200).WithLastBatchHeader(lastBatchNumALL).WithMoreBatchesExistHeader(false),
new PullKeysMockData(day3, 4, BatchType.ALL).HttpStatusCode(204),
new PullKeysMockData(day3, 5, BatchType.NO).HttpStatusCode(204),
});
//Given today is day2 AND that the new terms have not been approved.
SystemTime.SetDateTime(day3);
//Given last time we pulled was day1, batch1.
_helper.SetLastPulledDate(day1, 1);
//When pulling keys
List <string> zipLocations = (await new ZipDownloader().PullNewKeys(mockedService, new CancellationToken())).ToList();
//Then we pull all NO keys
Assert.Equal(8, zipLocations.Count);
Assert.Equal(lastBatchNumNO, LocalPreferencesHelper.LastPullKeysBatchNumberNotSubmitted); //The last batch number is saved from header
Assert.False((await _logManager.GetLogs(10)).Any()); //And no errors were logged
//Prepare for new pull
LocalPreferencesHelper.UpdateLastPullKeysSucceededDateTime();
await _logManager.DeleteAll();
_developerTools.ClearAllFields();
//When accepting the term
OnboardingStatusHelper.Status = OnboardingStatus.CountriesOnboardingCompleted;
//And then pull again
zipLocations = (await new ZipDownloader().PullNewKeys(mockedService, new CancellationToken())).ToList();
//Then we pull all EU keys but only for today
Assert.Equal(3, zipLocations.Count);
Assert.Equal(lastBatchNumALL, LocalPreferencesHelper.LastPullKeysBatchNumberNotSubmitted); //The last batch number is saved from header
Assert.False((await _logManager.GetLogs(10)).Any()); //And no errors were logged
//Next time it will also pull EU keys:
LocalPreferencesHelper.UpdateLastPullKeysSucceededDateTime();
PullKeysParams newParams = PullKeysParams.GenerateParams();
Assert.Equal(BatchType.ALL, newParams.BatchType);
}